From 2037ccae8d1b31175386c74dae857de782528960 Mon Sep 17 00:00:00 2001
From: Parman Mohammadalizadeh <prmma23@gmail.com>
Date: Sat, 1 Aug 2026 22:49:11 +0200
Subject: [PATCH 001/144] analyze: add --json output, fixes #9992

Every other status-type command (info, repo-info, repo-list, list, diff,
prune) can emit JSON, so tooling does not have to scrape their text. borg
analyze was the odd one out, although its numbers are exactly what
monitoring wants.

--json emits the numbers the text report is rendered from, as raw byte
values, for the default mode (dedup_size, hotspots) as well as for
--by-name (by_name). The compression factor is left out: it is
stored_size / source_size, and "n/a" is not a useful JSON value.

To keep one source of truth, the analysis methods now return their
numbers and the printing moved into report_*() methods that format them.
The text output is unchanged, byte for byte.

hotspots is null rather than empty when fewer than two archives matched:
the hot spots were then not computed at all, which is different from
having computed them and found nothing.
---
 docs/internals/frontends.rst                  |  78 ++++++++++
 docs/usage/analyze.rst.inc                    |  16 +-
 src/borg/archiver/_common.py                  |   1 +
 src/borg/archiver/analyze_cmd.py              | 147 ++++++++++++++----
 .../testsuite/archiver/analyze_cmd_test.py    | 127 ++++++++++++++-
 5 files changed, 339 insertions(+), 30 deletions(-)

diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst
index 9b9bbe8fb3..c42702bae2 100644
--- a/docs/internals/frontends.rst
+++ b/docs/internals/frontends.rst
@@ -561,6 +561,84 @@ Example (excerpt) of ``borg diff --json-lines``::
     {"path": "file3", "changes": [{"type": "removed", "size": 0}]}
 
 
+Archive Analysis
+++++++++++++++++
+
+:ref:`borg_analyze` ``--json`` emits the numbers of its text report as one object. All sizes are
+byte values; the compression factor the text report shows is ``stored_size / source_size``.
+
+Without ``--by-name``, the *dedup_size* and *hotspots* keys are present.
+
+*dedup_size* describes the considered set of archives:
+
+considered_archives
+    Number of archives matching the archive filters
+total_archives
+    Number of non-deleted archives in the repository
+whole_repository
+    True if no archive was left over by the filters, so the considered set is the whole
+    repository. Every referenced chunk is then trivially exclusive to the set, and the
+    *exclusive* key is absent.
+deduplicated
+    Object with *source_size* and *stored_size*: the summed size of the union of chunks the
+    considered archives reference, chunks shared within the set counted once
+exclusive
+    Object with *source_size* and *stored_size*: the chunks referenced only by the considered
+    set, i.e. what deleting the whole set would free. Absent if *whole_repository* is true.
+unreferenced
+    Object with *stored_size* and *chunks*: the chunks no non-deleted archive references, which
+    ``borg compact`` could free. Their source size is not known, as it is only recorded in the
+    archives referencing a chunk.
+total_chunks
+    Number of chunks in the repository chunk index
+missing_chunks
+    Number of chunks referenced by an archive but absent from the repository chunk index
+
+*hotspots* is a list of objects with *path* (directory path) and *size* (bytes of chunks added or
+removed in that directory between consecutive archives), busiest directory first. It is ``null``
+if fewer than two archives matched, as hot spots need at least two archives to compare.
+
+With ``--by-name``, the *by_name* key is present instead, decomposing the whole repository:
+
+archives
+    Number of non-deleted archives in the repository
+names
+    List of objects with *name*, *archives* (number of archives with that name), *source_size*
+    and *stored_size*. The sizes are what is exclusive to that name: no archive of another name
+    references those chunks. Biggest *stored_size* first.
+shared
+    Object with *source_size* and *stored_size*: the chunks referenced by two or more names
+unreferenced
+    As above
+total
+    Object with *archives*, *source_size* and *stored_size*. Each chunk is counted in exactly one
+    of *names*, *shared* and *unreferenced*, so the *names* and *shared* sizes add up to *total*.
+total_chunks, missing_chunks
+    As above
+
+Example of ``borg analyze -a 'sh:userA-*' --json``::
+
+    {
+        "dedup_size": {
+            "considered_archives": 2,
+            "deduplicated": {"source_size": 3000, "stored_size": 3536},
+            "exclusive": {"source_size": 2000, "stored_size": 3338},
+            "missing_chunks": 0,
+            "total_archives": 3,
+            "total_chunks": 13,
+            "unreferenced": {"chunks": 0, "stored_size": 0},
+            "whole_repository": false
+        },
+        "encryption": {"encryption": "aes256-ocb", "id_hash": "sha256"},
+        "hotspots": [{"path": "home/user/src", "size": 1000}],
+        "repository": {
+            "id": "06e4027d32f8eae8333f8fe06b1c2c46bf12f22ad10bd4d04a0f30751a26d77b",
+            "last_modified": "2026-08-01T22:46:05.886533",
+            "location": "/home/user/repository"
+        }
+    }
+
+
 .. _msgid:
 
 Message IDs
diff --git a/docs/usage/analyze.rst.inc b/docs/usage/analyze.rst.inc
index dd652d549b..91f2cfdeff 100644
--- a/docs/usage/analyze.rst.inc
+++ b/docs/usage/analyze.rst.inc
@@ -17,6 +17,8 @@ borg analyze
     +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+
     |                                                                             | ``--by-name``                                | decompose the whole repository by archive name (not combinable with archive filters)                                        |
     +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--json``                                   | format output as JSON                                                                                                       |
+    +-----------------------------------------------------------------------------+----------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+
     | .. class:: borg-common-opt-ref                                                                                                                                                                                                                           |
     |                                                                                                                                                                                                                                                          |
     | :ref:`common_options`                                                                                                                                                                                                                                    |
@@ -53,7 +55,8 @@ borg analyze
 
 
     options
-        --by-name     decompose the whole repository by archive name (not combinable with archive filters)
+        --by-name    decompose the whole repository by archive name (not combinable with archive filters)
+        --json     format output as JSON
 
 
     :ref:`common_options`
@@ -131,4 +134,13 @@ the sizes of added and removed chunks per direct parent directory, and outputs a
 You can use that list to find directories with a lot of "activity" — maybe some of these
 are temporary or cache directories you forgot to exclude. To avoid including these unwanted
 directories in your backups, you can carefully exclude them in ``borg create`` (for future
-backups) or use ``borg recreate`` to recreate existing archives without them.
\ No newline at end of file
+backups) or use ``borg recreate`` to recreate existing archives without them.
+
+**JSON output**
+
+With ``--json``, the same numbers are emitted as a single JSON object instead of the text
+report, with raw byte values rather than formatted sizes. The default mode fills the
+*dedup_size* and *hotspots* keys, ``--by-name`` fills the *by_name* key. The compression
+factor is not included: it is ``stored_size / source_size``.
+
+See :ref:`json_output` for the object's structure.
\ No newline at end of file
diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py
index 77787c26a5..6d3cee1089 100644
--- a/src/borg/archiver/_common.py
+++ b/src/borg/archiver/_common.py
@@ -280,6 +280,7 @@ def wrapper(self, args, repository, manifest, **kwargs):
     "key_files": "Internals -> Data structures and file formats -> Key files",
     "borg_key_export": "borg key export --help",
     "internals_hashindex": "Internals -> Data structures and file formats -> HashIndex",
+    "json_output": "Internals -> All about JSON: How to develop frontends",
 }
 
 
diff --git a/src/borg/archiver/analyze_cmd.py b/src/borg/archiver/analyze_cmd.py
index 55330d447b..639fb8a79b 100644
--- a/src/borg/archiver/analyze_cmd.py
+++ b/src/borg/archiver/analyze_cmd.py
@@ -5,7 +5,7 @@
 from ..archive import Archive
 from ..cache import get_archive_references, list_archive_reference_caches
 from ..constants import *  # NOQA
-from ..helpers import bin_to_hex, Error, format_file_size
+from ..helpers import basic_json_data, bin_to_hex, Error, format_file_size, json_print
 from ..helpers import ProgressIndicatorPercent
 from ..helpers.argparsing import ArgumentParser
 from ..manifest import Manifest
@@ -42,23 +42,40 @@ def __init__(self, args, repository, manifest):
 
     def analyze(self):
         logger.info("Starting archives analysis...")
+        json_data = {} if self.args.json else None
         if self.args.by_name:
             # the decomposition is inherently repository-wide: "shared" and "unreferenced" can only
             # be determined by looking at every archive, so archive filters must not be applied.
             filters = ["match_archives", "first", "last", "older", "newer", "oldest", "newest"]
             if any(getattr(self.args, name, None) for name in filters):
                 raise Error("--by-name analyzes the whole repository and cannot be combined with archive filters.")
-            self.analyze_by_name()
+            by_name = self.analyze_by_name()
+            if json_data is None:
+                self.report_by_name(by_name)
+            else:
+                json_data["by_name"] = by_name
         else:
             considered_infos = self.manifest.archives.list_considering(self.args)
             if not considered_infos:
                 raise Error("No archives match the given selection criteria.")
-            self.analyze_dedup_size(considered_infos)
+            dedup_size = self.analyze_dedup_size(considered_infos)
+            if json_data is None:
+                self.report_dedup_size(dedup_size)
+            else:
+                json_data["dedup_size"] = dedup_size
             if len(considered_infos) >= 2:
                 self.analyze_hotspots(considered_infos)
-                self.report_hotspots()
+                hotspots = self.hotspots()
             else:
                 logger.info("Skipping hot-spot analysis (needs at least 2 matching archives).")
+                hotspots = None  # not computed, as opposed to computed and empty
+            if json_data is None:
+                if hotspots is not None:
+                    self.report_hotspots(hotspots)
+            else:
+                json_data["hotspots"] = hotspots
+        if json_data is not None:
+            json_print(basic_json_data(self.manifest, extra=json_data))
         logger.info("Finished archives analysis.")
 
     def fmt(self, value):
@@ -101,7 +118,7 @@ def mark_references(self, archive_infos, update_flags):
         pi.finish()
         return missing
 
-    def analyze_by_name(self) -> None:
+    def analyze_by_name(self) -> dict:
         """Decompose the whole repository by archive name.
 
         Archives sharing a name form a series, so a name usually groups all backups of one source;
@@ -114,6 +131,8 @@ def analyze_by_name(self) -> None:
 
         This needs only a single pass over all archives: each chunk records the name that first
         referenced it, and a reference from a different name sets the F_MULTI bit.
+
+        Returns the decomposition as raw byte values, for report_by_name() or --json.
         """
         all_infos = self.manifest.archives.list()  # non-deleted archives
         if not all_infos:
@@ -159,6 +178,29 @@ def update_flags(flags, owner=owner):
 
         total_plaintext = shared[0] + sum(v[0] for v in exclusive.values())
         total_stored = shared[1] + sum(v[1] for v in exclusive.values())
+
+        return {
+            "archives": len(all_infos),
+            # biggest exclusive consumer first - that is what one would act on
+            "names": [
+                {
+                    "name": name,
+                    "archives": archives_per_name[name],
+                    "source_size": exclusive[name][0],
+                    "stored_size": exclusive[name][1],
+                }
+                for name in sorted(names, key=lambda n: exclusive[n][1], reverse=True)
+            ],
+            "shared": {"source_size": shared[0], "stored_size": shared[1]},
+            "unreferenced": {"stored_size": unref_stored, "chunks": unref_count},
+            "total": {"archives": len(all_infos), "source_size": total_plaintext, "stored_size": total_stored},
+            "total_chunks": total_count,
+            "missing_chunks": missing,
+        }
+
+    def report_by_name(self, data) -> None:
+        """Print the --by-name decomposition computed by analyze_by_name()."""
+        names = [entry["name"] for entry in data["names"]]
         width = min(max([30] + [len(name) for name in names]), 60)
 
         def row(label, archives, source, stored, *, source_known=True):
@@ -169,19 +211,24 @@ def row(label, archives, source, stored, *, source_known=True):
         print()
         print("Repository decomposition by archive name")
         print("=" * (width + 51))
-        print(f"{len(all_infos)} archive(s) with {len(names)} distinct name(s)")
+        print(f"{data['archives']} archive(s) with {len(names)} distinct name(s)")
         print()
         print(f"{'name':<{width}}{'archives':>10}{'source':>14}{'stored':>14}{'compression':>13}")
-        # biggest exclusive consumer first - that is what one would act on
-        for name in sorted(names, key=lambda n: exclusive[n][1], reverse=True):
-            source, stored = exclusive[name]
-            row(name, archives_per_name[name], source, stored)
-        row("(shared by 2+ names)", "", shared[0], shared[1])
-        row("(unreferenced)", "", 0, unref_stored, source_known=False)
+        for entry in data["names"]:
+            row(entry["name"], entry["archives"], entry["source_size"], entry["stored_size"])
+        row("(shared by 2+ names)", "", data["shared"]["source_size"], data["shared"]["stored_size"])
+        row("(unreferenced)", "", 0, data["unreferenced"]["stored_size"], source_known=False)
         print("-" * (width + 51))
-        row("total (deduplicated)", len(all_infos), total_plaintext, total_stored)
+        row(
+            "total (deduplicated)",
+            data["total"]["archives"],
+            data["total"]["source_size"],
+            data["total"]["stored_size"],
+        )
         print()
-        print(f"Unreferenced: {unref_count} of {total_count} chunks in the repository index.")
+        print(
+            f"Unreferenced: {data['unreferenced']['chunks']} of {data['total_chunks']} chunks in the repository index."
+        )
         print()
         print("Each chunk is counted in exactly one row, so the rows add up to the total.")
         print("A name row shows what is exclusive to it: no archive of another name references these")
@@ -197,8 +244,8 @@ def row(label, archives, source, stored, *, source_known=True):
         print(f"{'':<12}   archives count as unreferenced (borg compact keeps them if the")
         print(f"{'':<12}   repository is damaged).")
 
-    def analyze_dedup_size(self, considered_infos) -> None:
-        """Compute and report the deduplicated size of the considered set of archives.
+    def analyze_dedup_size(self, considered_infos) -> dict:
+        """Compute the deduplicated size of the considered set of archives.
 
         For both the plaintext (uncompressed source) size and the stored (compressed, as stored in
         the repository) size, two figures are reported:
@@ -221,6 +268,8 @@ def analyze_dedup_size(self, considered_infos) -> None:
         plaintext size (which is 0 in the repo index) from the per-archive references cache. These
         in-memory mutations are never persisted: write_chunkindex_to_repo() zeroes flags and size,
         and close() only serializes F_NEW entries (there are none here).
+
+        Returns the sizes as raw byte values, for report_dedup_size() or --json.
         """
         considered_ids = {info.id for info in considered_infos}
         all_infos = self.manifest.archives.list()  # non-deleted archives; the rest = all - considered
@@ -252,28 +301,54 @@ def analyze_dedup_size(self, considered_infos) -> None:
         # chunk is trivially exclusive to it, so that line would just repeat the deduplicated size.
         whole_repo = not rest_infos
 
+        data = {
+            "considered_archives": len(considered_infos),
+            "total_archives": len(all_infos),
+            "whole_repository": whole_repo,
+            "deduplicated": {"source_size": set_plaintext, "stored_size": set_stored},
+            "unreferenced": {"stored_size": unref_stored, "chunks": unref_count},
+            "total_chunks": total_count,
+            "missing_chunks": missing,
+        }
+        if not whole_repo:
+            data["exclusive"] = {"source_size": excl_plaintext, "stored_size": excl_stored}
+        return data
+
+    def report_dedup_size(self, data) -> None:
+        """Print the deduplicated sizes computed by analyze_dedup_size()."""
+        whole_repo = data["whole_repository"]
+
         def row(label, source, stored, *, source_known=True):
             sizes = f"{self.fmt(source) if source_known else 'n/a':>14}{self.fmt(stored):>14}"
             ratio = self.factor(stored, source) if source_known else "n/a"
             print(f"{label:<26}{sizes}{ratio:>13}")
 
+        considered = data["considered_archives"]
+        total_archives = data["total_archives"]
+        deduplicated = data["deduplicated"]
+        unreferenced = data["unreferenced"]
+
         print()
         if whole_repo:
             print("Deduplicated size of the whole repository")
             print("=" * 67)
-            print(f"Archives: {len(all_infos)} (all archives in the repository)")
+            print(f"Archives: {total_archives} (all archives in the repository)")
         else:
-            print(f"Deduplicated size of the {len(considered_infos)} considered archive(s)")
+            print(f"Deduplicated size of the {considered} considered archive(s)")
             print("=" * 67)
-            print(f"Considered archives: {len(considered_infos)} (of {len(all_infos)} in the repository)")
+            print(f"Considered archives: {considered} (of {total_archives} in the repository)")
         print()
         print(f"{'':26}{'source':>14}{'stored':>14}{'compression':>13}")
-        row("Deduplicated size:" if whole_repo else "Deduplicated size of set:", set_plaintext, set_stored)
+        row(
+            "Deduplicated size:" if whole_repo else "Deduplicated size of set:",
+            deduplicated["source_size"],
+            deduplicated["stored_size"],
+        )
         if not whole_repo:
-            row("Exclusive size of set:", excl_plaintext, excl_stored)
-        row("Unreferenced chunks:", 0, unref_stored, source_known=False)
+            row("Exclusive size of set:", data["exclusive"]["source_size"], data["exclusive"]["stored_size"])
+        row("Unreferenced chunks:", 0, unreferenced["stored_size"], source_known=False)
         print()
-        print(f"Unreferenced: {unref_count} of {total_count} chunks in the repository index.")
+        print(f"Unreferenced: {unreferenced['chunks']} of {data['total_chunks']} chunks in the repository index.")
         print()
         if whole_repo:
             print(f"{'source':<12} = uncompressed source data size (each chunk counted once)")
@@ -341,13 +416,21 @@ def analyze_path_change(path):
             if directory_path not in base:
                 analyze_path_change(directory_path)
 
-    def report_hotspots(self):
+    def hotspots(self) -> list:
+        """The hot spots collected by analyze_hotspots(), busiest directory first."""
+        return [
+            {"path": directory_path, "size": self.difference_by_path[directory_path]}
+            for directory_path in sorted(
+                self.difference_by_path, key=lambda p: self.difference_by_path[p], reverse=True
+            )
+        ]
+
+    def report_hotspots(self, hotspots):
         print()
         print("chunks added or removed by directory path")
         print("=========================================")
-        for directory_path in sorted(self.difference_by_path, key=lambda p: self.difference_by_path[p], reverse=True):
-            difference = self.difference_by_path[directory_path]
-            print(f"{directory_path}: {difference}")
+        for hotspot in hotspots:
+            print(f"{hotspot['path']}: {hotspot['size']}")
 
 
 class AnalyzeMixIn:
@@ -420,6 +503,15 @@ def build_parser_analyze(self, subparsers, common_parser, mid_common_parser):
             are temporary or cache directories you forgot to exclude. To avoid including these unwanted
             directories in your backups, you can carefully exclude them in ``borg create`` (for future
             backups) or use ``borg recreate`` to recreate existing archives without them.
+
+            **JSON output**
+
+            With ``--json``, the same numbers are emitted as a single JSON object instead of the text
+            report, with raw byte values rather than formatted sizes. The default mode fills the
+            *dedup_size* and *hotspots* keys, ``--by-name`` fills the *by_name* key. The compression
+            factor is not included: it is ``stored_size / source_size``.
+
+            See :ref:`json_output` for the object's structure.
             """
         )
         subparser = ArgumentParser(parents=[common_parser], description=self.do_analyze.__doc__, epilog=analyze_epilog)
@@ -430,4 +522,5 @@ def build_parser_analyze(self, subparsers, common_parser, mid_common_parser):
             action="store_true",
             help="decompose the whole repository by archive name (not combinable with archive filters)",
         )
+        subparser.add_argument("--json", action="store_true", help="format output as JSON")
         define_archive_filters_group(subparser)
diff --git a/src/borg/testsuite/archiver/analyze_cmd_test.py b/src/borg/testsuite/archiver/analyze_cmd_test.py
index 7afd3f3797..38d4967057 100644
--- a/src/borg/testsuite/archiver/analyze_cmd_test.py
+++ b/src/borg/testsuite/archiver/analyze_cmd_test.py
@@ -1,10 +1,11 @@
+import json
 import pathlib
 import re
 
 import pytest
 
 from ...constants import *  # NOQA
-from ...helpers import Error
+from ...helpers import Error, format_file_size
 from . import cmd, generate_archiver_tests, RK_ENCRYPTION
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local")  # NOQA
@@ -192,3 +193,127 @@ def test_analyze_dedup_size_single_archive(archivers, request):
     assert re.search(r"Deduplicated size of set:\s*1\.00 kB", output)
     # file1 is also in other-1, so nothing is exclusive to only-1
     assert re.search(r"Exclusive size of set:\s*0 B", output)
+
+
+def test_analyze_json(archivers, request):
+    """--json reports the same numbers as the text report, as raw byte values."""
+    archiver = request.getfixturevalue(archivers)
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    input_path = pathlib.Path(archiver.input_path)
+
+    # same layout as test_analyze_dedup_size: each file is one 1000 byte chunk.
+    (input_path / "shared").write_text("s" * 1000)
+    (input_path / "a_only").write_text("a" * 1000)
+    cmd(archiver, "create", "userA-1", archiver.input_path)
+    cmd(archiver, "create", "userA-2", archiver.input_path)
+
+    (input_path / "a_only").unlink()
+    (input_path / "b_only").write_text("b" * 1000)
+    cmd(archiver, "create", "userB-1", archiver.input_path)
+
+    json_output = cmd(archiver, "analyze", "-a", "sh:userA-*", "--json")
+    dedup_size = json.loads(json_output)["dedup_size"]
+    assert dedup_size["considered_archives"] == 2
+    assert dedup_size["total_archives"] == 3
+    assert dedup_size["whole_repository"] is False
+    assert dedup_size["deduplicated"]["source_size"] == 2000  # {shared, a_only}
+    assert dedup_size["exclusive"]["source_size"] == 1000  # {a_only}, "shared" is also in userB-1
+    assert dedup_size["unreferenced"] == {"stored_size": 0, "chunks": 0}
+    assert dedup_size["missing_chunks"] == 0
+    assert dedup_size["total_chunks"] > 0
+
+    # the stored sizes are compression dependent, so relate them to the text report instead of
+    # hardcoding them: both columns of a row are what the text report formats from these values.
+    text = cmd(archiver, "analyze", "-a", "sh:userA-*")
+    for key, label in [("deduplicated", "Deduplicated size of set:"), ("exclusive", "Exclusive size of set:")]:
+        source, stored = dedup_size[key]["source_size"], dedup_size[key]["stored_size"]
+        assert stored > 0
+        row = rf"^{re.escape(label)}\s+{re.escape(format_file_size(source))}\s+{re.escape(format_file_size(stored))}\s"
+        assert re.search(row, text, re.MULTILINE)
+
+    # the text report itself is not printed in JSON mode
+    assert "Deduplicated size of set:" not in json_output
+
+
+def test_analyze_json_whole_repository(archivers, request):
+    """Without an archive filter every chunk is trivially exclusive, so no exclusive size is given."""
+    archiver = request.getfixturevalue(archivers)
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    input_path = pathlib.Path(archiver.input_path)
+    (input_path / "file1").write_text("x" * 1000)
+    cmd(archiver, "create", "one", archiver.input_path)
+    (input_path / "file2").write_text("y" * 1000)
+    cmd(archiver, "create", "two", archiver.input_path)
+
+    dedup_size = json.loads(cmd(archiver, "analyze", "--json"))["dedup_size"]
+    assert dedup_size["whole_repository"] is True
+    assert dedup_size["considered_archives"] == 2
+    assert dedup_size["total_archives"] == 2
+    assert dedup_size["deduplicated"]["source_size"] == 2000
+    assert "exclusive" not in dedup_size
+
+
+def test_analyze_json_hotspots(archivers, request):
+    """The hot spots are a list of path/size objects, busiest first; null if they were not computed."""
+    archiver = request.getfixturevalue(archivers)
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    input_path = pathlib.Path(archiver.input_path)
+
+    (input_path / "file1").write_text("1")
+    cmd(archiver, "create", "archive", archiver.input_path)
+
+    # only one matching archive: nothing to compare against, so hot spots are not computed
+    assert json.loads(cmd(archiver, "analyze", "-a", "archive", "--json"))["hotspots"] is None
+
+    (input_path / "file2").write_text("22")
+    cmd(archiver, "create", "archive", archiver.input_path)
+
+    # the 2nd archive added one chunk of 2 bytes below the input directory
+    hotspots = json.loads(cmd(archiver, "analyze", "-a", "archive", "--json"))["hotspots"]
+    assert [hotspot for hotspot in hotspots if hotspot["path"].endswith("/input")] == [
+        {"path": str(input_path).removeprefix("/"), "size": 2}
+    ]
+    # busiest directory first, as in the text report
+    assert [hotspot["size"] for hotspot in hotspots] == sorted((hotspot["size"] for hotspot in hotspots), reverse=True)
+
+
+def test_analyze_json_by_name(archivers, request):
+    """--by-name --json decomposes the repository into per-name exclusive, shared and unreferenced."""
+    archiver = request.getfixturevalue(archivers)
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    input_path = pathlib.Path(archiver.input_path)
+
+    # same layout as test_analyze_by_name
+    (input_path / "shared").write_text("s" * 1000)
+    (input_path / "a_only").write_text("a" * 1000)
+    cmd(archiver, "create", "alpha", archiver.input_path)
+    cmd(archiver, "create", "alpha", archiver.input_path)
+
+    (input_path / "a_only").unlink()
+    (input_path / "b_only").write_text("b" * 1000)
+    cmd(archiver, "create", "beta", archiver.input_path)
+
+    result = json.loads(cmd(archiver, "analyze", "--by-name", "--json"))
+    assert "dedup_size" not in result and "hotspots" not in result
+    by_name = result["by_name"]
+
+    assert by_name["archives"] == 3
+    assert {entry["name"]: entry["archives"] for entry in by_name["names"]} == {"alpha": 2, "beta": 1}
+    assert {entry["name"]: entry["source_size"] for entry in by_name["names"]} == {"alpha": 1000, "beta": 1000}
+    assert by_name["shared"]["source_size"] == 1000
+    assert by_name["total"]["archives"] == 3
+    assert by_name["total"]["source_size"] == 3000  # 1000 alpha + 1000 beta + 1000 shared
+    assert by_name["total"]["stored_size"] > 0
+    assert by_name["missing_chunks"] == 0
+
+    # every chunk is counted in exactly one row, so the rows add up to the total
+    for size in ("source_size", "stored_size"):
+        assert sum(entry[size] for entry in by_name["names"]) + by_name["shared"][size] == by_name["total"][size]
+    # biggest exclusive consumer first
+    assert [entry["stored_size"] for entry in by_name["names"]] == sorted(
+        (entry["stored_size"] for entry in by_name["names"]), reverse=True
+    )

From 05509440228ea84a5e7b6059a41dae4a65b50462 Mon Sep 17 00:00:00 2001
From: Parman Mohammadalizadeh <prmma23@gmail.com>
Date: Mon, 3 Aug 2026 08:53:04 +0200
Subject: [PATCH 002/144] analyze: fix the hot-spot JSON test on Windows

The test rebuilt the expected hot-spot path from the input directory,
stripping a leading slash. Archived paths are normalized, and on Windows
that also drops the drive colon (C:\Users -> C/Users), so the expectation
read D:/a/... where borg had stored D/a/....

Assert the size of the input directory's hot spot by path suffix, like
the text-report test above already does, and check the paths against
what the text report prints instead of rebuilding them.
---
 src/borg/testsuite/archiver/analyze_cmd_test.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/borg/testsuite/archiver/analyze_cmd_test.py b/src/borg/testsuite/archiver/analyze_cmd_test.py
index 38d4967057..df49bfc124 100644
--- a/src/borg/testsuite/archiver/analyze_cmd_test.py
+++ b/src/borg/testsuite/archiver/analyze_cmd_test.py
@@ -273,9 +273,13 @@ def test_analyze_json_hotspots(archivers, request):
 
     # the 2nd archive added one chunk of 2 bytes below the input directory
     hotspots = json.loads(cmd(archiver, "analyze", "-a", "archive", "--json"))["hotspots"]
-    assert [hotspot for hotspot in hotspots if hotspot["path"].endswith("/input")] == [
-        {"path": str(input_path).removeprefix("/"), "size": 2}
-    ]
+    assert [hotspot["size"] for hotspot in hotspots if hotspot["path"].endswith("/input")] == [2]
+    # paths and sizes are the ones the text report prints. Archived paths are normalized
+    # (a Windows "D:/x" is stored as "D/x"), so compare against the report rather than
+    # rebuilding the path here.
+    text = cmd(archiver, "analyze", "-a", "archive")
+    for hotspot in hotspots:
+        assert f"{hotspot['path']}: {hotspot['size']}" in text
     # busiest directory first, as in the text report
     assert [hotspot["size"] for hotspot in hotspots] == sorted((hotspot["size"] for hotspot in hotspots), reverse=True)
 

From afedfb682b1b44777a05ff62d99316461e0009e3 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Tue, 21 Jul 2026 19:48:26 +0530
Subject: [PATCH 003/144] check: keep the records of corrupt packs across check
 cycles

Only the intact-pack records are cycle progress; the corrupt ones are
kept for repair until the pack verifies intact or is gone from packs/,
and their ids are now reported in the check summary. Refs #9696.
---
 docs/internals/data-structures.rst    |  8 ++-
 src/borg/repository.py                | 57 +++++++++++++++----
 src/borg/testsuite/repository_test.py | 79 ++++++++++++++++++++++++++-
 3 files changed, 128 insertions(+), 16 deletions(-)

diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index 53bb619efd..39b6294787 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -36,9 +36,11 @@ config/
 
 cache/
   checked-packs
-    repository check progress (partial checks, full checks' checkpointing),
-    the set of packs checked so far this cycle (pack id -> timestamp, result),
-    as a hashtable with an appended integrity hash
+    repository check results (pack id -> timestamp, result), as a hashtable with an
+    appended integrity hash. Records of intact packs hold the check progress (partial
+    checks, full checks' checkpointing) and are dropped when a check cycle completes.
+    Records of corrupt packs are kept for repair until the pack verifies intact or is
+    no longer listed in packs/.
 
 There is a list of pointers to archive objects in this directory:
 
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 254c094589..2a126d4545 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -313,11 +313,14 @@ def superseded_gap_ranges(reader, chunks, pack_id, obj_ranges, pack_size):
 
 
 class PackTracker:
-    """Packs verified in the current check cycle, mapping pack_id -> (timestamp, result).
+    """Pack verification results, mapping pack_id -> (timestamp, result).
 
     A cycle is one full pass over packs/; --max-duration may spread it over several partial checks.
+    Intact records (result=1) hold the cycle progress and are dropped when the cycle completes.
+    Corrupt records (result=0) are kept across cycles until the pack verifies intact or is no longer
+    listed in packs/.
     Stored at cache/checked-packs as the serialized table with a sha256 over it appended.
-    new() starts a cycle, load() resumes the stored one.
+    new() starts an empty tracker, load() reads the stored one.
     """
 
     NAME = "cache/checked-packs"
@@ -377,6 +380,31 @@ def get(self, pack_id):
     def record(self, pack_id, ok):
         self.table[pack_id] = self.Entry(timestamp=int(time.time()), result=int(ok))
 
+    def corrupt_ids(self):
+        """Return the ids of the packs recorded corrupt, sorted."""
+        return sorted(pack_id for pack_id, entry in self.table.items() if not entry.result)
+
+    def drop_ok(self):
+        """Remove the intact-pack records, keeping the corrupt ones."""
+        # the keys are collected first because the table must not be mutated while iterating it.
+        for pack_id in [pack_id for pack_id, entry in self.table.items() if entry.result]:
+            del self.table[pack_id]
+
+    def finish_cycle(self, pack_ids):
+        """End a completed cycle: drop the intact records and the corrupt records of packs that are
+        gone, then store the remaining records (or delete the stored object if none remain).
+
+        pack_ids is the set of pack ids listed in packs/ during this cycle; a record for an id not in
+        it refers to a pack that was deleted or compacted away.
+        """
+        self.drop_ok()
+        for pack_id in [pack_id for pack_id, _ in self.table.items() if pack_id not in pack_ids]:
+            del self.table[pack_id]
+        if len(self.table):
+            self.save()
+        else:
+            self.clear()
+
     def save(self):
         with io.BytesIO() as f:
             self.table.write(f)
@@ -815,7 +843,8 @@ def check(self, repair=False, max_duration=0):
         rebuild re-reads every pack anyway - so a read-only check just stops and reports it instead of
         continuing. The index is never rebuilt here in any case: reading every pack to do so would be
         far too slow and expensive for a routine (e.g. cron) check. Salvaging good objects out of
-        corrupt packs and dropping those packs is left to repair, refs #8572.
+        corrupt packs and dropping those packs is left to repair, refs #8572. The ids of the packs
+        found corrupt are kept in cache/checked-packs for repair, refs #9696.
         """
 
         def verify(namespace, name):
@@ -839,15 +868,15 @@ def store_list(namespace):
         assert not (repair and partial)
         mode = "partial" if partial else "full"
         logger.info(f"Starting {mode} repository check")
-        if partial:
-            tracker = PackTracker.load(self.store)
-        else:
-            tracker = PackTracker.new(self.store)
-            tracker.clear()  # a full check verifies every pack, so discard the stored cycle
-        if len(tracker):
+        tracker = PackTracker.load(self.store)
+        if not partial:
+            tracker.drop_ok()  # a full check verifies every pack, so it starts a new cycle
+        if not len(tracker):
+            logger.info("Starting from beginning.")
+        elif partial:
             logger.info(f"Continuing check cycle, {len(tracker)} packs already checked.")
         else:
-            logger.info("Starting from beginning.")
+            logger.info(f"Starting from beginning, re-verifying {len(tracker)} packs recorded corrupt.")
         t_start = time.monotonic()
         t_last_checkpoint = t_start
         index_files = index_errors = 0
@@ -899,11 +928,11 @@ def store_list(namespace):
                     tracker.save()
                     break
             else:
-                # scanned all packs without hitting the time limit: the cycle is done, drop the set.
+                # scanned all packs without hitting the time limit: the cycle is done, drop its progress.
                 if pack_infos:
                     pack_pi.show(current=len(pack_infos))  # finish at 100%
                 logger.info("Finished checking packs.")
-                tracker.clear()
+                tracker.finish_cycle({hex_to_bin(info.name) for info in pack_infos})
             pack_pi.finish()
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
@@ -912,6 +941,10 @@ def store_list(namespace):
         logger.info(
             f"Checked {index_files} index files ({index_errors} errors) and {pack_files} packs ({pack_errors} errors)."
         )
+        if index_errors == 0:  # the packs were checked, so the corrupt records are from this check
+            corrupt_ids = tracker.corrupt_ids()
+            if corrupt_ids:
+                logger.error("Corrupt packs: " + ", ".join(bin_to_hex(pack_id) for pack_id in corrupt_ids))
         if objs_errors == 0:
             logger.info(f"Finished {mode} repository check, no problems found.")
         elif repair:
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 1e6d7cfd1e..217ceb95ec 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1,4 +1,5 @@
 import io
+import logging
 import os
 import sys
 from collections import namedtuple
@@ -1133,7 +1134,83 @@ def test_check_full_ignores_recorded_set(tmp_path, monkeypatch):
         assert pack_key in hashed_keys  # verified
 
         after = PackTracker.load(repository.store)
-        assert len(after) == 0  # cycle complete, set dropped
+        assert len(after) == 0  # cycle complete, intact records dropped
+
+
+def _store_corrupt_pack(repository, pack_id):
+    # the stored content does not hash to pack_id, so verifying it fails.
+    repository.store_store("packs/" + bin_to_hex(pack_id), b"CORRUPT-does-not-match-name")
+    return pack_id
+
+
+def test_check_full_keeps_corrupt_record_after_cycle(tmp_path):
+    # a completed cycle keeps the records of corrupt packs and drops the records of intact ones.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, _ = _store_intact_pack(repository)
+        corrupt_id = _store_corrupt_pack(repository, H(2))
+
+        assert repository.check(repair=False) is False
+
+        after = PackTracker.load(repository.store)
+        assert after.corrupt_ids() == [corrupt_id]
+        assert intact_id not in after.table
+
+
+def test_check_full_reports_corrupt_pack_ids(tmp_path, caplog):
+    # the summary names the corrupt packs.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        corrupt_id = _store_corrupt_pack(repository, H(1))
+
+        with caplog.at_level(logging.ERROR, logger="borg.repository"):
+            assert repository.check(repair=False) is False
+
+        assert f"Corrupt packs: {bin_to_hex(corrupt_id)}" in caplog.text
+
+
+def test_check_full_reverifies_carried_over_corrupt_record(tmp_path, monkeypatch):
+    # a corrupt record from an earlier cycle is re-verified and dropped once the pack is intact again.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        tracker.record(intact_id, ok=False)  # recorded corrupt in an earlier cycle
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False) is True
+        assert pack_key in hashed_keys  # re-verified
+
+        after = PackTracker.load(repository.store)
+        assert len(after) == 0  # verified intact, so the corrupt record is dropped
+
+
+def test_check_full_prunes_corrupt_record_of_vanished_pack(tmp_path):
+    # a corrupt record for a pack that is no longer listed is dropped at cycle end.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        tracker.record(H(9), ok=False)  # no such pack in packs/
+        tracker.save()
+
+        assert repository.check(repair=False) is True
+
+        after = PackTracker.load(repository.store)
+        assert len(after) == 0
+
+
+def test_check_partial_keeps_corrupt_record_across_runs(tmp_path):
+    # corrupt records survive a partial check that completes its cycle, too.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        corrupt_id = _store_corrupt_pack(repository, H(1))
+
+        assert repository.check(repair=False, max_duration=3600) is False
+        assert PackTracker.load(repository.store).corrupt_ids() == [corrupt_id]
+
+        # a second run re-verifies it and keeps reporting it.
+        assert repository.check(repair=False, max_duration=3600) is False
+        assert PackTracker.load(repository.store).corrupt_ids() == [corrupt_id]
 
 
 def test_check_checked_packs_ignores_foreign_entry_layout(tmp_path):

From 4e3372e6b13f1a7a1a5a9dd08f1a23186507ca19 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 23 Jul 2026 02:14:27 +0530
Subject: [PATCH 004/144] check: add --max-age to reuse intact-pack check
 results across cycles

---
 docs/internals/data-structures.rst            |  6 +-
 src/borg/archiver/check_cmd.py                | 22 ++++-
 src/borg/repository.py                        | 35 ++++---
 src/borg/testsuite/archiver/check_cmd_test.py | 19 ++++
 src/borg/testsuite/repository_test.py         | 94 +++++++++++++++++++
 5 files changed, 158 insertions(+), 18 deletions(-)

diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index 39b6294787..e62a04c4da 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -39,8 +39,10 @@ cache/
     repository check results (pack id -> timestamp, result), as a hashtable with an
     appended integrity hash. Records of intact packs hold the check progress (partial
     checks, full checks' checkpointing) and are dropped when a check cycle completes.
-    Records of corrupt packs are kept for repair until the pack verifies intact or is
-    no longer listed in packs/.
+    With ``check --max-age`` they are kept across cycles instead and reused while
+    younger than the given age. Records of corrupt packs are kept for repair until
+    the pack verifies intact or is no longer listed in packs/. Records of packs no
+    longer listed in packs/ are pruned when a cycle completes.
 
 There is a list of pointers to archive objects in this directory:
 
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 682f3a9279..c0d1ea1b7e 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -4,7 +4,7 @@
 from ..archive import ArchiveChecker
 from ..constants import *  # NOQA
 from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError
-from ..helpers import yes, ArchiveFormatter
+from ..helpers import interval, yes, ArchiveFormatter
 from ..helpers.argparsing import ArgumentParser
 
 from ..logger import create_logger
@@ -44,6 +44,8 @@ def do_check(self, args, repository):
             raise CommandError("--repository-only contradicts the --find-lost-archives option.")
         if args.repair and args.max_duration:
             raise CommandError("--repair does not allow --max-duration argument.")
+        if args.repair and args.max_age:
+            raise CommandError("--repair does not allow the --max-age option.")
         if args.max_duration and not args.repo_only:
             # when doing a partial repo check, we can only do a low-level check of the repository files.
             # archives check requires that a full repo check was done before and has built/cached a ChunkIndex.
@@ -64,7 +66,8 @@ def do_check(self, args, repository):
             # the repository check has finished, which can take hours.
             ArchiveFormatter.validate_format(format)
         if not args.archives_only:
-            if not repository.check(repair=args.repair, max_duration=args.max_duration):
+            max_age = int(args.max_age.total_seconds()) if args.max_age else 0
+            if not repository.check(repair=args.repair, max_duration=args.max_duration, max_age=max_age):
                 set_ec(EXIT_WARNING)
         if not args.repo_only and not archive_checker.check(
             repository,
@@ -130,6 +133,12 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         ``--max-duration`` you must also pass ``--repository-only``, and must not pass
         ``--archives-only``, nor ``--repair``.
 
+        The ``--max-age`` option keeps the results of previous repository checks and
+        skips packs whose intact result is younger than the given interval (e.g.
+        ``--max-age=4w``), spreading the verification cost over repeated checks.
+        Packs recorded corrupt are always re-verified. ``--max-age`` cannot be
+        combined with ``--repair``.
+
         **Warning:** Please note that partial repository checks (i.e., running with
         ``--max-duration``) can only perform non-cryptographic checksum checks on the
         repository files. Enabling partial repository checks excludes archive checks
@@ -222,6 +231,15 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         subparser.add_argument(
             "--find-lost-archives", dest="find_lost_archives", action="store_true", help="attempt to find lost archives"
         )
+        subparser.add_argument(
+            "--max-age",
+            metavar="INTERVAL",
+            dest="max_age",
+            type=interval,
+            default=None,
+            action=Highlander,
+            help="reuse intact-pack check results younger than INTERVAL (e.g. 4w)",
+        )
         subparser.add_argument(
             "--max-duration",
             metavar="SECONDS",
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 2a126d4545..c3011c9eba 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -316,9 +316,9 @@ class PackTracker:
     """Pack verification results, mapping pack_id -> (timestamp, result).
 
     A cycle is one full pass over packs/; --max-duration may spread it over several partial checks.
-    Intact records (result=1) hold the cycle progress and are dropped when the cycle completes.
-    Corrupt records (result=0) are kept across cycles until the pack verifies intact or is no longer
-    listed in packs/.
+    Intact records (result=1) hold the cycle progress and are dropped when the cycle completes,
+    or kept across cycles when the cycle is finished with keep_ok. Corrupt records (result=0)
+    are kept across cycles until the pack verifies intact or is no longer listed in packs/.
     Stored at cache/checked-packs as the serialized table with a sha256 over it appended.
     new() starts an empty tracker, load() reads the stored one.
     """
@@ -390,14 +390,15 @@ def drop_ok(self):
         for pack_id in [pack_id for pack_id, entry in self.table.items() if entry.result]:
             del self.table[pack_id]
 
-    def finish_cycle(self, pack_ids):
-        """End a completed cycle: drop the intact records and the corrupt records of packs that are
-        gone, then store the remaining records (or delete the stored object if none remain).
+    def finish_cycle(self, pack_ids, keep_ok=False):
+        """End a completed cycle: drop the intact records (unless keep_ok) and the records of packs
+        that are gone, then store the remaining records (or delete the stored object if none remain).
 
         pack_ids is the set of pack ids listed in packs/ during this cycle; a record for an id not in
         it refers to a pack that was deleted or compacted away.
         """
-        self.drop_ok()
+        if not keep_ok:
+            self.drop_ok()
         for pack_id in [pack_id for pack_id, _ in self.table.items() if pack_id not in pack_ids]:
             del self.table[pack_id]
         if len(self.table):
@@ -831,7 +832,7 @@ def info(self):
         info = dict(id=self.id, version=self.version)
         return info
 
-    def check(self, repair=False, max_duration=0):
+    def check(self, repair=False, max_duration=0, max_age=0):
         """Check repository consistency.
 
         packs/ and index/ objects are named by the sha256 of their content, so a pack or index file
@@ -845,6 +846,9 @@ def check(self, repair=False, max_duration=0):
         far too slow and expensive for a routine (e.g. cron) check. Salvaging good objects out of
         corrupt packs and dropping those packs is left to repair, refs #8572. The ids of the packs
         found corrupt are kept in cache/checked-packs for repair, refs #9696.
+
+        max_age (seconds, 0 = off): keep the intact-pack records across cycles and skip packs whose
+        intact record is younger than max_age.
         """
 
         def verify(namespace, name):
@@ -869,12 +873,14 @@ def store_list(namespace):
         mode = "partial" if partial else "full"
         logger.info(f"Starting {mode} repository check")
         tracker = PackTracker.load(self.store)
-        if not partial:
-            tracker.drop_ok()  # a full check verifies every pack, so it starts a new cycle
+        if not partial and not max_age:
+            tracker.drop_ok()  # without max_age, a full check verifies every pack: start a new cycle
         if not len(tracker):
             logger.info("Starting from beginning.")
         elif partial:
-            logger.info(f"Continuing check cycle, {len(tracker)} packs already checked.")
+            logger.info(f"Continuing check cycle, {len(tracker)} pack check results on record.")
+        elif max_age:
+            logger.info(f"{len(tracker)} pack check results on record, reusing those younger than max_age.")
         else:
             logger.info(f"Starting from beginning, re-verifying {len(tracker)} packs recorded corrupt.")
         t_start = time.monotonic()
@@ -910,8 +916,9 @@ def store_list(namespace):
                 pack_pi.show(increase=1)  # advance for skipped packs too, so the bar tracks packs/, not work done
                 pack_id = hex_to_bin(info.name)
                 entry = tracker.get(pack_id)
-                if entry is not None and entry.result:  # intact in this cycle; a corrupt one is verified again
-                    continue
+                if entry is not None and entry.result:  # recorded intact; a corrupt one is verified again
+                    if not max_age or time.time() - entry.timestamp < max_age:
+                        continue
                 pack_files += 1
                 ok = verify("packs", info.name)
                 if not ok:
@@ -932,7 +939,7 @@ def store_list(namespace):
                 if pack_infos:
                     pack_pi.show(current=len(pack_infos))  # finish at 100%
                 logger.info("Finished checking packs.")
-                tracker.finish_cycle({hex_to_bin(info.name) for info in pack_infos})
+                tracker.finish_cycle({hex_to_bin(info.name) for info in pack_infos}, keep_ok=bool(max_age))
             pack_pi.finish()
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index ef42ad776b..102ebbca15 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -74,6 +74,25 @@ def test_check_usage(archivers, request):
     assert "archive2" in output
 
 
+def test_check_max_age(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+
+    # --repair does not allow --max-age.
+    if archiver.FORK_DEFAULT:
+        cmd(archiver, "check", "--repair", "--max-age=1d", exit_code=CommandError().exit_code)
+    else:
+        with pytest.raises(CommandError):
+            cmd(archiver, "check", "--repair", "--max-age=1d")
+
+    # a first check with --max-age records the results, a second one reuses them.
+    output = cmd(archiver, "check", "-v", "--repository-only", "--max-age=4w", exit_code=0)
+    assert "Starting full repository check" in output
+    output = cmd(archiver, "check", "-v", "--repository-only", "--max-age=4w", exit_code=0)
+    assert "reusing those younger than max_age" in output
+    assert "no problems found" in output
+
+
 def test_date_matching(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 217ceb95ec..590763a40d 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -2,6 +2,7 @@
 import logging
 import os
 import sys
+import time
 from collections import namedtuple
 from hashlib import sha256
 
@@ -1213,6 +1214,99 @@ def test_check_partial_keeps_corrupt_record_across_runs(tmp_path):
         assert PackTracker.load(repository.store).corrupt_ids() == [corrupt_id]
 
 
+def test_check_max_age_skips_fresh_ok(tmp_path, monkeypatch):
+    # with max_age, a pack recorded intact recently is not re-verified, and its record is kept.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        tracker.record(intact_id, ok=True)  # fresh timestamp
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=3600) is True
+        assert pack_key not in hashed_keys  # skipped, its record is fresh
+
+        after = PackTracker.load(repository.store)
+        assert after.table[intact_id].result == 1  # kept across the cycle
+
+
+def test_check_max_age_reverifies_stale_ok(tmp_path, monkeypatch):
+    # with max_age, a pack whose intact record is older than max_age is re-verified.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        old_ts = int(time.time()) - 100
+        tracker.table[intact_id] = PackTracker.Entry(timestamp=old_ts, result=1)
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=50) is True
+        assert pack_key in hashed_keys  # stale, re-verified
+
+        after = PackTracker.load(repository.store)
+        assert after.table[intact_id].timestamp > old_ts  # record refreshed
+
+
+def test_check_max_age_reverifies_corrupt_even_when_fresh(tmp_path, monkeypatch):
+    # the age window only applies to intact records: a fresh corrupt record is still re-verified.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        tracker.record(intact_id, ok=False)  # fresh, but corrupt
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=3600) is True
+        assert pack_key in hashed_keys  # re-verified despite the fresh record
+
+
+def test_check_max_age_prunes_vanished_ok_record(tmp_path):
+    # with max_age, an intact record for a pack no longer listed is pruned at cycle end.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, _ = _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        tracker.record(intact_id, ok=True)
+        tracker.record(H(9), ok=True)  # no such pack in packs/
+        tracker.save()
+
+        assert repository.check(repair=False, max_age=3600) is True
+
+        after = PackTracker.load(repository.store)
+        assert intact_id in after.table
+        assert H(9) not in after.table
+
+
+def test_check_max_age_partial_progress(tmp_path, monkeypatch):
+    # a partial check with max_age skips packs with a fresh intact record and verifies the rest.
+    pack_a = fchunk(b"A", chunk_id=H(1))
+    pack_a_id = sha256(pack_a).digest()
+    pack_b = fchunk(b"BB", chunk_id=H(2))
+    pack_b_id = sha256(pack_b).digest()
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        repository.store_store("packs/" + bin_to_hex(pack_a_id), pack_a)
+        repository.store_store("packs/" + bin_to_hex(pack_b_id), pack_b)
+
+        tracker = PackTracker.new(repository.store)
+        tracker.record(pack_a_id, ok=True)  # fresh
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_duration=3600, max_age=3600) is True
+        assert "packs/" + bin_to_hex(pack_a_id) not in hashed_keys
+        assert "packs/" + bin_to_hex(pack_b_id) in hashed_keys
+
+        after = PackTracker.load(repository.store)
+        assert pack_a_id in after.table and pack_b_id in after.table
+
+
 def test_check_checked_packs_ignores_foreign_entry_layout(tmp_path):
     # load() drops a set whose entries have a different layout than Entry, even though its sha256 matches.
     OtherEntry = namedtuple("OtherEntry", "timestamp result extra")

From cf00471de4d9b830cdc13886fae4092d3846141a Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Fri, 24 Jul 2026 04:12:44 +0530
Subject: [PATCH 005/144] check: keep pack check results unconditionally,
 --max-age only controls their reuse

Records are pruned only for packs no longer listed in packs/, so --max-duration now requires --max-age to make progress.
---
 docs/internals/data-structures.rst            | 12 ++--
 src/borg/archiver/check_cmd.py                | 37 ++++++-----
 src/borg/repository.py                        | 43 ++++---------
 src/borg/testsuite/archiver/check_cmd_test.py |  9 ++-
 src/borg/testsuite/repository_test.py         | 62 ++++++++++++-------
 5 files changed, 84 insertions(+), 79 deletions(-)

diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index e62a04c4da..3637448311 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -37,12 +37,12 @@ config/
 cache/
   checked-packs
     repository check results (pack id -> timestamp, result), as a hashtable with an
-    appended integrity hash. Records of intact packs hold the check progress (partial
-    checks, full checks' checkpointing) and are dropped when a check cycle completes.
-    With ``check --max-age`` they are kept across cycles instead and reused while
-    younger than the given age. Records of corrupt packs are kept for repair until
-    the pack verifies intact or is no longer listed in packs/. Records of packs no
-    longer listed in packs/ are pruned when a cycle completes.
+    appended integrity hash. Records are kept across checks: ``check --max-age``
+    skips packs whose intact record is younger than the given age, which also lets
+    partial checks (``--max-duration``) continue where a previous one stopped.
+    Records of corrupt packs are kept for repair and always re-verified. Records of
+    packs no longer listed in packs/ are pruned when a check finishes scanning
+    packs/.
 
 There is a list of pointers to archive objects in this directory:
 
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index c0d1ea1b7e..d0ce135417 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -46,6 +46,9 @@ def do_check(self, args, repository):
             raise CommandError("--repair does not allow --max-duration argument.")
         if args.repair and args.max_age:
             raise CommandError("--repair does not allow the --max-age option.")
+        if args.max_duration and not args.max_age:
+            # partial checks progress by skipping packs whose record is younger than max_age.
+            raise CommandError("--max-duration requires the --max-age option.")
         if args.max_duration and not args.repo_only:
             # when doing a partial repo check, we can only do a low-level check of the repository files.
             # archives check requires that a full repo check was done before and has built/cached a ChunkIndex.
@@ -121,23 +124,25 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         repository checks only, or pass ``--archives-only`` to run the archive checks
         only.
 
-        The ``--max-duration`` option can be used to split a long-running repository
-        check into multiple partial checks. After the given number of seconds, the check
-        is interrupted. The next partial check will continue where the previous one
-        stopped, until the full repository has been checked. Assuming a complete check
-        would take 7 hours, then running a daily check with ``--max-duration=3600``
-        (1 hour) would result in one full repository check per week. Doing a full
-        repository check aborts any previous partial check; the next partial check will
-        restart from the beginning. With partial repository checks you can run neither
-        archive checks, nor enable repair mode. Consequently, if you want to use
-        ``--max-duration`` you must also pass ``--repository-only``, and must not pass
-        ``--archives-only``, nor ``--repair``.
+        The ``--max-age`` option makes the check reuse the results of previous
+        repository checks: packs whose intact result is younger than the given
+        interval (e.g. ``--max-age=4w``) are skipped, spreading the verification
+        cost over repeated checks. Check results are recorded in any case;
+        ``--max-age`` only controls their reuse. Packs recorded corrupt are always
+        re-verified. ``--max-age`` cannot be combined with ``--repair``.
 
-        The ``--max-age`` option keeps the results of previous repository checks and
-        skips packs whose intact result is younger than the given interval (e.g.
-        ``--max-age=4w``), spreading the verification cost over repeated checks.
-        Packs recorded corrupt are always re-verified. ``--max-age`` cannot be
-        combined with ``--repair``.
+        The ``--max-duration`` option can be used to split a long-running repository
+        check into multiple partial checks. After the given number of seconds, the
+        check is interrupted. Because a verified pack's result is recorded and
+        reused, ``--max-duration`` requires ``--max-age``: the next partial check
+        skips the recently verified packs and continues with the rest, until every
+        pack has a result younger than ``--max-age``. Assuming a complete check
+        would take 7 hours, then running a daily check with ``--max-duration=3600
+        --max-age=1w`` (1 hour) would result in one full repository verification
+        per week. With partial repository checks you can run neither archive
+        checks, nor enable repair mode. Consequently, if you want to use
+        ``--max-duration`` you must also pass ``--repository-only``, and must not
+        pass ``--archives-only``, nor ``--repair``.
 
         **Warning:** Please note that partial repository checks (i.e., running with
         ``--max-duration``) can only perform non-cryptographic checksum checks on the
diff --git a/src/borg/repository.py b/src/borg/repository.py
index c3011c9eba..8ab3035e87 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -315,10 +315,9 @@ def superseded_gap_ranges(reader, chunks, pack_id, obj_ranges, pack_size):
 class PackTracker:
     """Pack verification results, mapping pack_id -> (timestamp, result).
 
-    A cycle is one full pass over packs/; --max-duration may spread it over several partial checks.
-    Intact records (result=1) hold the cycle progress and are dropped when the cycle completes,
-    or kept across cycles when the cycle is finished with keep_ok. Corrupt records (result=0)
-    are kept across cycles until the pack verifies intact or is no longer listed in packs/.
+    Records are kept across checks: intact records (result=1) are reused by checks run with
+    max_age, corrupt records (result=0) are kept for repair and always re-verified. Records of
+    packs no longer listed in packs/ are pruned when a check finishes scanning packs/.
     Stored at cache/checked-packs as the serialized table with a sha256 over it appended.
     new() starts an empty tracker, load() reads the stored one.
     """
@@ -384,21 +383,11 @@ def corrupt_ids(self):
         """Return the ids of the packs recorded corrupt, sorted."""
         return sorted(pack_id for pack_id, entry in self.table.items() if not entry.result)
 
-    def drop_ok(self):
-        """Remove the intact-pack records, keeping the corrupt ones."""
-        # the keys are collected first because the table must not be mutated while iterating it.
-        for pack_id in [pack_id for pack_id, entry in self.table.items() if entry.result]:
-            del self.table[pack_id]
-
-    def finish_cycle(self, pack_ids, keep_ok=False):
-        """End a completed cycle: drop the intact records (unless keep_ok) and the records of packs
-        that are gone, then store the remaining records (or delete the stored object if none remain).
-
-        pack_ids is the set of pack ids listed in packs/ during this cycle; a record for an id not in
-        it refers to a pack that was deleted or compacted away.
+    def prune(self, pack_ids):
+        """Drop the records whose pack id is not in pack_ids (the set of pack ids listed in packs/),
+        then store the remaining records (or delete the stored object if none remain).
         """
-        if not keep_ok:
-            self.drop_ok()
+        # the keys are collected first because the table must not be mutated while iterating it.
         for pack_id in [pack_id for pack_id, _ in self.table.items() if pack_id not in pack_ids]:
             del self.table[pack_id]
         if len(self.table):
@@ -847,8 +836,8 @@ def check(self, repair=False, max_duration=0, max_age=0):
         corrupt packs and dropping those packs is left to repair, refs #8572. The ids of the packs
         found corrupt are kept in cache/checked-packs for repair, refs #9696.
 
-        max_age (seconds, 0 = off): keep the intact-pack records across cycles and skip packs whose
-        intact record is younger than max_age.
+        max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
+        max_age. Check results are always recorded and kept.
         """
 
         def verify(namespace, name):
@@ -873,16 +862,12 @@ def store_list(namespace):
         mode = "partial" if partial else "full"
         logger.info(f"Starting {mode} repository check")
         tracker = PackTracker.load(self.store)
-        if not partial and not max_age:
-            tracker.drop_ok()  # without max_age, a full check verifies every pack: start a new cycle
         if not len(tracker):
             logger.info("Starting from beginning.")
-        elif partial:
-            logger.info(f"Continuing check cycle, {len(tracker)} pack check results on record.")
         elif max_age:
             logger.info(f"{len(tracker)} pack check results on record, reusing those younger than max_age.")
         else:
-            logger.info(f"Starting from beginning, re-verifying {len(tracker)} packs recorded corrupt.")
+            logger.info(f"{len(tracker)} pack check results on record, verifying every pack.")
         t_start = time.monotonic()
         t_last_checkpoint = t_start
         index_files = index_errors = 0
@@ -917,7 +902,7 @@ def store_list(namespace):
                 pack_id = hex_to_bin(info.name)
                 entry = tracker.get(pack_id)
                 if entry is not None and entry.result:  # recorded intact; a corrupt one is verified again
-                    if not max_age or time.time() - entry.timestamp < max_age:
+                    if max_age and time.time() - entry.timestamp < max_age:
                         continue
                 pack_files += 1
                 ok = verify("packs", info.name)
@@ -931,15 +916,13 @@ def store_list(namespace):
                     logger.info(f"Checkpointing at pack {info.name}.")
                     tracker.save()
                 if partial and now > t_start + max_duration:
-                    logger.info(f"Finished partial repository check, {len(tracker)} packs checked so far.")
-                    tracker.save()
+                    logger.info(f"Finished partial repository check, {len(tracker)} pack check results on record.")
                     break
             else:
-                # scanned all packs without hitting the time limit: the cycle is done, drop its progress.
                 if pack_infos:
                     pack_pi.show(current=len(pack_infos))  # finish at 100%
                 logger.info("Finished checking packs.")
-                tracker.finish_cycle({hex_to_bin(info.name) for info in pack_infos}, keep_ok=bool(max_age))
+            tracker.prune({hex_to_bin(info.name) for info in pack_infos})
             pack_pi.finish()
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 102ebbca15..fd82333764 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -78,15 +78,18 @@ def test_check_max_age(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
 
-    # --repair does not allow --max-age.
+    # --repair does not allow --max-age, and --max-duration requires --max-age.
     if archiver.FORK_DEFAULT:
         cmd(archiver, "check", "--repair", "--max-age=1d", exit_code=CommandError().exit_code)
+        cmd(archiver, "check", "--repository-only", "--max-duration=3600", exit_code=CommandError().exit_code)
     else:
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--repair", "--max-age=1d")
+        with pytest.raises(CommandError):
+            cmd(archiver, "check", "--repository-only", "--max-duration=3600")
 
-    # a first check with --max-age records the results, a second one reuses them.
-    output = cmd(archiver, "check", "-v", "--repository-only", "--max-age=4w", exit_code=0)
+    # a check records its results, a later one with --max-age reuses them.
+    output = cmd(archiver, "check", "-v", "--repository-only", exit_code=0)
     assert "Starting full repository check" in output
     output = cmd(archiver, "check", "-v", "--repository-only", "--max-age=4w", exit_code=0)
     assert "reusing those younger than max_age" in output
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 590763a40d..c192e12249 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1043,7 +1043,7 @@ def test_check_partial_rechecks_pack_sorting_before_checked_one(tmp_path):
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         repository.store_store("packs/" + bin_to_hex(intact_id), intact)
 
-        # mark the intact pack as already checked in this cycle.
+        # mark the intact pack as recently checked.
         tracker = PackTracker.new(repository.store)
         tracker.record(intact_id, ok=True)
         tracker.save()
@@ -1053,11 +1053,11 @@ def test_check_partial_rechecks_pack_sorting_before_checked_one(tmp_path):
         assert bin_to_hex(early_id) < bin_to_hex(intact_id)
         repository.store_store("packs/" + bin_to_hex(early_id), b"CORRUPT-does-not-match-name")
 
-        assert repository.check(repair=False, max_duration=3600) is False
+        assert repository.check(repair=False, max_duration=3600, max_age=3600) is False
 
 
 def test_check_partial_rechecks_pack_recorded_corrupt(tmp_path):
-    # a pack recorded corrupt earlier in the cycle is re-verified, so the corruption keeps being reported.
+    # a pack recorded corrupt earlier is re-verified, so the corruption keeps being reported.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         corrupt_id = H(1)  # stored content does not hash to this name
         repository.store_store("packs/" + bin_to_hex(corrupt_id), b"CORRUPT-does-not-match-name")
@@ -1066,7 +1066,7 @@ def test_check_partial_rechecks_pack_recorded_corrupt(tmp_path):
         tracker.record(corrupt_id, ok=False)
         tracker.save()
 
-        assert repository.check(repair=False, max_duration=3600) is False
+        assert repository.check(repair=False, max_duration=3600, max_age=3600) is False
 
 
 def _spy_hash(repository, monkeypatch):
@@ -1101,12 +1101,12 @@ def test_check_partial_clears_recorded_corruption_when_intact(tmp_path, monkeypa
 
         hashed_keys = _spy_hash(repository, monkeypatch)
 
-        assert repository.check(repair=False, max_duration=3600) is True
+        assert repository.check(repair=False, max_duration=3600, max_age=3600) is True
         assert pack_key in hashed_keys  # re-verified
 
 
 def test_check_partial_skips_pack_recorded_intact(tmp_path, monkeypatch):
-    # a pack recorded intact in this cycle is skipped when a partial check resumes.
+    # a pack recorded intact recently is skipped when a partial check resumes.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
@@ -1116,12 +1116,12 @@ def test_check_partial_skips_pack_recorded_intact(tmp_path, monkeypatch):
 
         hashed_keys = _spy_hash(repository, monkeypatch)
 
-        assert repository.check(repair=False, max_duration=3600) is True
+        assert repository.check(repair=False, max_duration=3600, max_age=3600) is True
         assert pack_key not in hashed_keys  # skipped, not re-verified
 
 
 def test_check_full_ignores_recorded_set(tmp_path, monkeypatch):
-    # a full check verifies every pack regardless of the recorded set, then drops the set.
+    # without max_age, a check verifies every pack regardless of the recorded set, but keeps the records.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
@@ -1132,10 +1132,10 @@ def test_check_full_ignores_recorded_set(tmp_path, monkeypatch):
         hashed_keys = _spy_hash(repository, monkeypatch)
 
         assert repository.check(repair=False) is True
-        assert pack_key in hashed_keys  # verified
+        assert pack_key in hashed_keys  # verified despite the fresh intact record
 
         after = PackTracker.load(repository.store)
-        assert len(after) == 0  # cycle complete, intact records dropped
+        assert after.table[intact_id].result == 1  # record kept
 
 
 def _store_corrupt_pack(repository, pack_id):
@@ -1144,8 +1144,8 @@ def _store_corrupt_pack(repository, pack_id):
     return pack_id
 
 
-def test_check_full_keeps_corrupt_record_after_cycle(tmp_path):
-    # a completed cycle keeps the records of corrupt packs and drops the records of intact ones.
+def test_check_full_keeps_records_after_check(tmp_path):
+    # a completed check keeps the records of both corrupt and intact packs.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, _ = _store_intact_pack(repository)
         corrupt_id = _store_corrupt_pack(repository, H(2))
@@ -1154,7 +1154,7 @@ def test_check_full_keeps_corrupt_record_after_cycle(tmp_path):
 
         after = PackTracker.load(repository.store)
         assert after.corrupt_ids() == [corrupt_id]
-        assert intact_id not in after.table
+        assert after.table[intact_id].result == 1
 
 
 def test_check_full_reports_corrupt_pack_ids(tmp_path, caplog):
@@ -1169,12 +1169,12 @@ def test_check_full_reports_corrupt_pack_ids(tmp_path, caplog):
 
 
 def test_check_full_reverifies_carried_over_corrupt_record(tmp_path, monkeypatch):
-    # a corrupt record from an earlier cycle is re-verified and dropped once the pack is intact again.
+    # a corrupt record from an earlier check is re-verified; an intact result replaces it.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
         tracker = PackTracker.new(repository.store)
-        tracker.record(intact_id, ok=False)  # recorded corrupt in an earlier cycle
+        tracker.record(intact_id, ok=False)  # recorded corrupt in an earlier check
         tracker.save()
 
         hashed_keys = _spy_hash(repository, monkeypatch)
@@ -1183,13 +1183,13 @@ def test_check_full_reverifies_carried_over_corrupt_record(tmp_path, monkeypatch
         assert pack_key in hashed_keys  # re-verified
 
         after = PackTracker.load(repository.store)
-        assert len(after) == 0  # verified intact, so the corrupt record is dropped
+        assert after.table[intact_id].result == 1  # verified intact, corrupt record replaced
 
 
 def test_check_full_prunes_corrupt_record_of_vanished_pack(tmp_path):
-    # a corrupt record for a pack that is no longer listed is dropped at cycle end.
+    # a corrupt record for a pack that is no longer listed is dropped when the check finishes.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
-        _store_intact_pack(repository)
+        intact_id, _ = _store_intact_pack(repository)
 
         tracker = PackTracker.new(repository.store)
         tracker.record(H(9), ok=False)  # no such pack in packs/
@@ -1198,19 +1198,20 @@ def test_check_full_prunes_corrupt_record_of_vanished_pack(tmp_path):
         assert repository.check(repair=False) is True
 
         after = PackTracker.load(repository.store)
-        assert len(after) == 0
+        assert H(9) not in after.table
+        assert intact_id in after.table
 
 
 def test_check_partial_keeps_corrupt_record_across_runs(tmp_path):
-    # corrupt records survive a partial check that completes its cycle, too.
+    # corrupt records survive completed partial checks, too.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         corrupt_id = _store_corrupt_pack(repository, H(1))
 
-        assert repository.check(repair=False, max_duration=3600) is False
+        assert repository.check(repair=False, max_duration=3600, max_age=3600) is False
         assert PackTracker.load(repository.store).corrupt_ids() == [corrupt_id]
 
         # a second run re-verifies it and keeps reporting it.
-        assert repository.check(repair=False, max_duration=3600) is False
+        assert repository.check(repair=False, max_duration=3600, max_age=3600) is False
         assert PackTracker.load(repository.store).corrupt_ids() == [corrupt_id]
 
 
@@ -1229,7 +1230,7 @@ def test_check_max_age_skips_fresh_ok(tmp_path, monkeypatch):
         assert pack_key not in hashed_keys  # skipped, its record is fresh
 
         after = PackTracker.load(repository.store)
-        assert after.table[intact_id].result == 1  # kept across the cycle
+        assert after.table[intact_id].result == 1  # record kept
 
 
 def test_check_max_age_reverifies_stale_ok(tmp_path, monkeypatch):
@@ -1267,7 +1268,7 @@ def test_check_max_age_reverifies_corrupt_even_when_fresh(tmp_path, monkeypatch)
 
 
 def test_check_max_age_prunes_vanished_ok_record(tmp_path):
-    # with max_age, an intact record for a pack no longer listed is pruned at cycle end.
+    # an intact record for a pack no longer listed is pruned when the check finishes.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, _ = _store_intact_pack(repository)
 
@@ -1307,6 +1308,19 @@ def test_check_max_age_partial_progress(tmp_path, monkeypatch):
         assert pack_a_id in after.table and pack_b_id in after.table
 
 
+def test_check_max_age_reuses_records_of_plain_check(tmp_path, monkeypatch):
+    # a check without max_age records its results, a later check with max_age reuses them.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        assert repository.check(repair=False) is True
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=3600) is True
+        assert pack_key not in hashed_keys  # skipped, reusing the plain check's record
+
+
 def test_check_checked_packs_ignores_foreign_entry_layout(tmp_path):
     # load() drops a set whose entries have a different layout than Entry, even though its sha256 matches.
     OtherEntry = namedtuple("OtherEntry", "timestamp result extra")

From 556daa23c7a4738dcac8798a6ad4e0c7d8dc68e4 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Tue, 28 Jul 2026 23:03:15 +0530
Subject: [PATCH 006/144] check: re-verify future-dated records, reject
 --archives-only --max-age, log corrupt packs one per line

---
 docs/internals/data-structures.rst    |  3 +--
 src/borg/archiver/check_cmd.py        |  5 ++++-
 src/borg/constants.py                 |  5 +++++
 src/borg/repository.py                | 12 +++++++++---
 src/borg/testsuite/repository_test.py |  4 ++--
 5 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index 3637448311..048a798500 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -41,8 +41,7 @@ cache/
     skips packs whose intact record is younger than the given age, which also lets
     partial checks (``--max-duration``) continue where a previous one stopped.
     Records of corrupt packs are kept for repair and always re-verified. Records of
-    packs no longer listed in packs/ are pruned when a check finishes scanning
-    packs/.
+    packs no longer listed in packs/ are pruned when a check finishes.
 
 There is a list of pointers to archive objects in this directory:
 
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index d0ce135417..30413389c8 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -46,9 +46,12 @@ def do_check(self, args, repository):
             raise CommandError("--repair does not allow --max-duration argument.")
         if args.repair and args.max_age:
             raise CommandError("--repair does not allow the --max-age option.")
+        if args.archives_only and args.max_age:
+            # --max-age only affects the repository check, which --archives-only skips.
+            raise CommandError("--archives-only does not allow the --max-age option.")
         if args.max_duration and not args.max_age:
             # partial checks progress by skipping packs whose record is younger than max_age.
-            raise CommandError("--max-duration requires the --max-age option.")
+            raise CommandError("--max-duration requires the --max-age option, e.g. --max-age=4w.")
         if args.max_duration and not args.repo_only:
             # when doing a partial repo check, we can only do a low-level check of the repository files.
             # archives check requires that a full repo check was done before and has built/cached a ChunkIndex.
diff --git a/src/borg/constants.py b/src/borg/constants.py
index 42b8ea07ce..7d3874a9c6 100644
--- a/src/borg/constants.py
+++ b/src/borg/constants.py
@@ -71,6 +71,11 @@
 # MAX_OBJECT_SIZE = MAX_DATA_SIZE + len(PUT header)
 MAX_OBJECT_SIZE = MAX_DATA_SIZE + 41  # see assertion at end of repository module
 
+# Clock skew is the difference between the clocks of the machines writing to a repository (seconds).
+# A check result timestamp up to this far in the future still counts as recent; further ahead than
+# this, the pack is re-verified.
+MAX_CLOCK_SKEW = 7200  # [s]
+
 # How many segment files Borg puts into a single directory by default.
 DEFAULT_SEGMENTS_PER_DIR = 1000
 
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 8ab3035e87..c9de5c4d72 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -901,8 +901,11 @@ def store_list(namespace):
                 pack_pi.show(increase=1)  # advance for skipped packs too, so the bar tracks packs/, not work done
                 pack_id = hex_to_bin(info.name)
                 entry = tracker.get(pack_id)
-                if entry is not None and entry.result:  # recorded intact; a corrupt one is verified again
-                    if max_age and time.time() - entry.timestamp < max_age:
+                # skip the pack if its recorded intact result is younger than max_age; a timestamp up
+                # to MAX_CLOCK_SKEW in the future still counts as recent, further ahead it is re-verified.
+                if entry is not None and entry.result and max_age:
+                    age = time.time() - entry.timestamp
+                    if -MAX_CLOCK_SKEW <= age < max_age:
                         continue
                 pack_files += 1
                 ok = verify("packs", info.name)
@@ -934,7 +937,10 @@ def store_list(namespace):
         if index_errors == 0:  # the packs were checked, so the corrupt records are from this check
             corrupt_ids = tracker.corrupt_ids()
             if corrupt_ids:
-                logger.error("Corrupt packs: " + ", ".join(bin_to_hex(pack_id) for pack_id in corrupt_ids))
+                # one id per line (the list can be long).
+                logger.error(f"Found {len(corrupt_ids)} corrupt pack(s):")
+                for pack_id in corrupt_ids:
+                    logger.error(f"Corrupt pack: {bin_to_hex(pack_id)}")
         if objs_errors == 0:
             logger.info(f"Finished {mode} repository check, no problems found.")
         elif repair:
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index c192e12249..47a1004e76 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1120,7 +1120,7 @@ def test_check_partial_skips_pack_recorded_intact(tmp_path, monkeypatch):
         assert pack_key not in hashed_keys  # skipped, not re-verified
 
 
-def test_check_full_ignores_recorded_set(tmp_path, monkeypatch):
+def test_check_without_max_age_verifies_all_but_keeps_records(tmp_path, monkeypatch):
     # without max_age, a check verifies every pack regardless of the recorded set, but keeps the records.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
@@ -1165,7 +1165,7 @@ def test_check_full_reports_corrupt_pack_ids(tmp_path, caplog):
         with caplog.at_level(logging.ERROR, logger="borg.repository"):
             assert repository.check(repair=False) is False
 
-        assert f"Corrupt packs: {bin_to_hex(corrupt_id)}" in caplog.text
+        assert f"Corrupt pack: {bin_to_hex(corrupt_id)}" in caplog.text
 
 
 def test_check_full_reverifies_carried_over_corrupt_record(tmp_path, monkeypatch):

From e2c03cb8ba8739f60bdb12bde9d480dbc127b8b0 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Tue, 28 Jul 2026 23:33:04 +0530
Subject: [PATCH 007/144] check: fail and report known-corrupt packs even on
 interrupted partial checks, show reused-result count

---
 src/borg/repository.py                | 32 +++++++++++++++++----------
 src/borg/testsuite/repository_test.py | 32 +++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 12 deletions(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index c9de5c4d72..7686c7a354 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -871,7 +871,7 @@ def store_list(namespace):
         t_start = time.monotonic()
         t_last_checkpoint = t_start
         index_files = index_errors = 0
-        pack_files = pack_errors = 0
+        pack_files = pack_errors = pack_skipped = 0
         # index and packs get separate progress indicators, each running from 0% to 100%.
         # the index is checked first and in full, on partial checks too: it is small, and index errors
         # stop the pack check below.
@@ -906,6 +906,7 @@ def store_list(namespace):
                 if entry is not None and entry.result and max_age:
                     age = time.time() - entry.timestamp
                     if -MAX_CLOCK_SKEW <= age < max_age:
+                        pack_skipped += 1
                         continue
                 pack_files += 1
                 ok = verify("packs", info.name)
@@ -931,23 +932,30 @@ def store_list(namespace):
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
         objs_errors = index_errors + pack_errors
-        logger.info(
-            f"Checked {index_files} index files ({index_errors} errors) and {pack_files} packs ({pack_errors} errors)."
+        summary = (
+            f"Checked {index_files} index files ({index_errors} errors) "
+            f"and {pack_files} packs ({pack_errors} errors)."
         )
-        if index_errors == 0:  # the packs were checked, so the corrupt records are from this check
-            corrupt_ids = tracker.corrupt_ids()
-            if corrupt_ids:
-                # one id per line (the list can be long).
-                logger.error(f"Found {len(corrupt_ids)} corrupt pack(s):")
-                for pack_id in corrupt_ids:
-                    logger.error(f"Corrupt pack: {bin_to_hex(pack_id)}")
-        if objs_errors == 0:
+        if pack_skipped:
+            summary += f" Reused {pack_skipped} recent pack check result(s)."
+        logger.info(summary)
+        # every pack recorded corrupt, not only those verified this run; empty if the index is
+        # corrupt, since then no packs were scanned.
+        corrupt_ids = tracker.corrupt_ids() if index_errors == 0 else []
+        if corrupt_ids:
+            # one id per line (the list can be long).
+            logger.error(f"Found {len(corrupt_ids)} corrupt pack(s):")
+            for pack_id in corrupt_ids:
+                logger.error(f"Corrupt pack: {bin_to_hex(pack_id)}")
+        # fail if this run found errors, or any pack is recorded corrupt.
+        problems = objs_errors != 0 or bool(corrupt_ids)
+        if not problems:
             logger.info(f"Finished {mode} repository check, no problems found.")
         elif repair:
             logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).")
         else:
             logger.error(f"Finished {mode} repository check, errors found.")
-        return objs_errors == 0 or repair
+        return not problems or repair
 
     def list(self, limit=None, marker=None):
         """
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 47a1004e76..f074d3ade6 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1215,6 +1215,38 @@ def test_check_partial_keeps_corrupt_record_across_runs(tmp_path):
         assert PackTracker.load(repository.store).corrupt_ids() == [corrupt_id]
 
 
+def test_check_partial_break_reports_unreached_corrupt_record(tmp_path, monkeypatch, caplog):
+    # a partial check that stops before re-reaching a carried-over corrupt record still fails and
+    # reports it, so the timeout does not hide known corruption.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, _ = _store_intact_pack(repository)
+        # a corrupt pack whose id sorts after the intact one, so the scan reaches the intact pack first.
+        corrupt_id = b"\xff" * 32
+        assert bin_to_hex(intact_id) < bin_to_hex(corrupt_id)
+        repository.store_store("packs/" + bin_to_hex(corrupt_id), b"CORRUPT-does-not-match-name")
+
+        tracker = PackTracker.new(repository.store)
+        tracker.record(corrupt_id, ok=False)  # recorded corrupt by an earlier check
+        tracker.save()
+
+        # jump the clock past max_duration once the first pack has been hashed, so the scan breaks
+        # before it reaches the corrupt pack.
+        clock = {"t": 0}
+        monkeypatch.setattr(time, "monotonic", lambda: clock["t"])
+        orig_hash = repository.store.hash
+
+        def hash_then_advance(key):
+            result = orig_hash(key)
+            clock["t"] = 10**9
+            return result
+
+        monkeypatch.setattr(repository.store, "hash", hash_then_advance)
+
+        with caplog.at_level(logging.ERROR, logger="borg.repository"):
+            assert repository.check(repair=False, max_duration=1, max_age=3600) is False
+        assert f"Corrupt pack: {bin_to_hex(corrupt_id)}" in caplog.text
+
+
 def test_check_max_age_skips_fresh_ok(tmp_path, monkeypatch):
     # with max_age, a pack recorded intact recently is not re-verified, and its record is kept.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:

From d46523109664e815655751007230ef46cdf7770a Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 29 Jul 2026 13:26:40 +0530
Subject: [PATCH 008/144] check: test clock-skew reuse window and archives-only
 rejection, assert partial break skips corrupt pack

---
 src/borg/archiver/check_cmd.py                |  3 +-
 src/borg/testsuite/archiver/check_cmd_test.py |  5 +-
 src/borg/testsuite/repository_test.py         | 60 +++++++++++++++----
 3 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 30413389c8..333ccefa0b 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -132,7 +132,8 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         interval (e.g. ``--max-age=4w``) are skipped, spreading the verification
         cost over repeated checks. Check results are recorded in any case;
         ``--max-age`` only controls their reuse. Packs recorded corrupt are always
-        re-verified. ``--max-age`` cannot be combined with ``--repair``.
+        re-verified. ``--max-age`` affects only the repository check and cannot be
+        combined with ``--archives-only`` or ``--repair``.
 
         The ``--max-duration`` option can be used to split a long-running repository
         check into multiple partial checks. After the given number of seconds, the
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index fd82333764..fb225f1d37 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -78,13 +78,16 @@ def test_check_max_age(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
 
-    # --repair does not allow --max-age, and --max-duration requires --max-age.
+    # --repair and --archives-only do not allow --max-age, and --max-duration requires --max-age.
     if archiver.FORK_DEFAULT:
         cmd(archiver, "check", "--repair", "--max-age=1d", exit_code=CommandError().exit_code)
+        cmd(archiver, "check", "--archives-only", "--max-age=1d", exit_code=CommandError().exit_code)
         cmd(archiver, "check", "--repository-only", "--max-duration=3600", exit_code=CommandError().exit_code)
     else:
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--repair", "--max-age=1d")
+        with pytest.raises(CommandError):
+            cmd(archiver, "check", "--archives-only", "--max-age=1d")
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--repository-only", "--max-duration=3600")
 
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index f074d3ade6..059b252e4b 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -9,6 +9,7 @@
 import pytest
 from borghash import HashTableNT
 
+from ..constants import MAX_CLOCK_SKEW
 from ..helpers import IntegrityError, Location, bin_to_hex
 from ..hashindex import ChunkIndex
 from ..repository import Repository, MAX_DATA_SIZE, propagate_rsh, rest_serve_command, PackWriter, PackReader
@@ -1216,34 +1217,39 @@ def test_check_partial_keeps_corrupt_record_across_runs(tmp_path):
 
 
 def test_check_partial_break_reports_unreached_corrupt_record(tmp_path, monkeypatch, caplog):
-    # a partial check that stops before re-reaching a carried-over corrupt record still fails and
-    # reports it, so the timeout does not hide known corruption.
+    # a partial check that breaks before re-reaching a corrupt record from an earlier check still
+    # fails and reports that pack.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
-        intact_id, _ = _store_intact_pack(repository)
+        intact_id, intact_key = _store_intact_pack(repository)
         # a corrupt pack whose id sorts after the intact one, so the scan reaches the intact pack first.
         corrupt_id = b"\xff" * 32
         assert bin_to_hex(intact_id) < bin_to_hex(corrupt_id)
-        repository.store_store("packs/" + bin_to_hex(corrupt_id), b"CORRUPT-does-not-match-name")
+        corrupt_key = "packs/" + bin_to_hex(corrupt_id)
+        repository.store_store(corrupt_key, b"CORRUPT-does-not-match-name")
 
         tracker = PackTracker.new(repository.store)
         tracker.record(corrupt_id, ok=False)  # recorded corrupt by an earlier check
         tracker.save()
 
-        # jump the clock past max_duration once the first pack has been hashed, so the scan breaks
-        # before it reaches the corrupt pack.
-        clock = {"t": 0}
+        # freeze the clock, then jump it past max_duration right after the intact pack is hashed, so the
+        # pack loop breaks before it reaches the corrupt pack.
+        clock = {"t": 0.0}
         monkeypatch.setattr(time, "monotonic", lambda: clock["t"])
+        hashed_keys = []
         orig_hash = repository.store.hash
 
-        def hash_then_advance(key):
+        def hash_and_advance(key):
+            hashed_keys.append(key)
             result = orig_hash(key)
-            clock["t"] = 10**9
+            if key == intact_key:
+                clock["t"] = 10**9
             return result
 
-        monkeypatch.setattr(repository.store, "hash", hash_then_advance)
+        monkeypatch.setattr(repository.store, "hash", hash_and_advance)
 
         with caplog.at_level(logging.ERROR, logger="borg.repository"):
             assert repository.check(repair=False, max_duration=1, max_age=3600) is False
+        assert corrupt_key not in hashed_keys  # the scan broke before reaching the corrupt pack
         assert f"Corrupt pack: {bin_to_hex(corrupt_id)}" in caplog.text
 
 
@@ -1284,6 +1290,40 @@ def test_check_max_age_reverifies_stale_ok(tmp_path, monkeypatch):
         assert after.table[intact_id].timestamp > old_ts  # record refreshed
 
 
+def test_check_max_age_skips_near_future_ok(tmp_path, monkeypatch):
+    # a record timestamped slightly in the future (clock skew between machines) still counts as
+    # recent, up to MAX_CLOCK_SKEW ahead, and is not re-verified.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        future_ts = int(time.time()) + MAX_CLOCK_SKEW // 2
+        tracker.table[intact_id] = PackTracker.Entry(timestamp=future_ts, result=1)
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=3600) is True
+        assert pack_key not in hashed_keys  # near-future timestamp still counts as recent
+
+
+def test_check_max_age_reverifies_far_future_ok(tmp_path, monkeypatch):
+    # a record dated more than MAX_CLOCK_SKEW into the future is not plausible clock skew and is
+    # re-verified.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        tracker = PackTracker.new(repository.store)
+        far_future_ts = int(time.time()) + MAX_CLOCK_SKEW + 3600
+        tracker.table[intact_id] = PackTracker.Entry(timestamp=far_future_ts, result=1)
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=3600) is True
+        assert pack_key in hashed_keys  # too far ahead to be clock skew, re-verified
+
+
 def test_check_max_age_reverifies_corrupt_even_when_fresh(tmp_path, monkeypatch):
     # the age window only applies to intact records: a fresh corrupt record is still re-verified.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:

From 8ec9224bfe0d54b72fc9a4369761ab10e9d4a651 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 29 Jul 2026 13:37:40 +0530
Subject: [PATCH 009/144] check: cap max-age future-skew tolerance at the
 window, document persistent corrupt-pack failure

---
 src/borg/repository.py                | 11 ++++++---
 src/borg/testsuite/repository_test.py | 32 +++++++++++++++++++++------
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index 7686c7a354..fe3566edcf 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -836,6 +836,10 @@ def check(self, repair=False, max_duration=0, max_age=0):
         corrupt packs and dropping those packs is left to repair, refs #8572. The ids of the packs
         found corrupt are kept in cache/checked-packs for repair, refs #9696.
 
+        Any pack on record as corrupt fails the check, including on a partial run that stops before
+        re-reaching it. The record clears when the pack verifies intact again, when compact removes
+        the pack, or when repair salvages and drops it (refs #8572).
+
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
         max_age. Check results are always recorded and kept.
         """
@@ -901,11 +905,12 @@ def store_list(namespace):
                 pack_pi.show(increase=1)  # advance for skipped packs too, so the bar tracks packs/, not work done
                 pack_id = hex_to_bin(info.name)
                 entry = tracker.get(pack_id)
-                # skip the pack if its recorded intact result is younger than max_age; a timestamp up
-                # to MAX_CLOCK_SKEW in the future still counts as recent, further ahead it is re-verified.
+                # skip the pack if its recorded intact result is younger than max_age. a future
+                # timestamp (writer clock ahead of ours) also counts as recent, tolerated up to
+                # MAX_CLOCK_SKEW or max_age ahead, whichever is smaller.
                 if entry is not None and entry.result and max_age:
                     age = time.time() - entry.timestamp
-                    if -MAX_CLOCK_SKEW <= age < max_age:
+                    if -min(MAX_CLOCK_SKEW, max_age) <= age < max_age:
                         pack_skipped += 1
                         continue
                 pack_files += 1
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 059b252e4b..467722caca 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1291,8 +1291,8 @@ def test_check_max_age_reverifies_stale_ok(tmp_path, monkeypatch):
 
 
 def test_check_max_age_skips_near_future_ok(tmp_path, monkeypatch):
-    # a record timestamped slightly in the future (clock skew between machines) still counts as
-    # recent, up to MAX_CLOCK_SKEW ahead, and is not re-verified.
+    # with a window wider than MAX_CLOCK_SKEW, a record timestamped up to MAX_CLOCK_SKEW into the
+    # future (writer clock ahead of ours) still counts as recent and is not re-verified.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
@@ -1303,13 +1303,13 @@ def test_check_max_age_skips_near_future_ok(tmp_path, monkeypatch):
 
         hashed_keys = _spy_hash(repository, monkeypatch)
 
-        assert repository.check(repair=False, max_age=3600) is True
-        assert pack_key not in hashed_keys  # near-future timestamp still counts as recent
+        assert repository.check(repair=False, max_age=MAX_CLOCK_SKEW * 2) is True
+        assert pack_key not in hashed_keys  # within MAX_CLOCK_SKEW ahead, still counts as recent
 
 
 def test_check_max_age_reverifies_far_future_ok(tmp_path, monkeypatch):
-    # a record dated more than MAX_CLOCK_SKEW into the future is not plausible clock skew and is
-    # re-verified.
+    # with a window wider than MAX_CLOCK_SKEW, a record more than MAX_CLOCK_SKEW into the future is
+    # not plausible clock skew and is re-verified.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
@@ -1320,10 +1320,28 @@ def test_check_max_age_reverifies_far_future_ok(tmp_path, monkeypatch):
 
         hashed_keys = _spy_hash(repository, monkeypatch)
 
-        assert repository.check(repair=False, max_age=3600) is True
+        assert repository.check(repair=False, max_age=MAX_CLOCK_SKEW * 2) is True
         assert pack_key in hashed_keys  # too far ahead to be clock skew, re-verified
 
 
+def test_check_max_age_reverifies_future_beyond_small_window(tmp_path, monkeypatch):
+    # the future tolerance is capped at max_age: with a window smaller than MAX_CLOCK_SKEW, a record
+    # further ahead than max_age is re-verified even though it is within MAX_CLOCK_SKEW.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        small_window = MAX_CLOCK_SKEW // 4
+        tracker = PackTracker.new(repository.store)
+        future_ts = int(time.time()) + MAX_CLOCK_SKEW // 2  # ahead of us, but < MAX_CLOCK_SKEW
+        tracker.table[intact_id] = PackTracker.Entry(timestamp=future_ts, result=1)
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=small_window) is True
+        assert pack_key in hashed_keys  # further ahead than max_age, re-verified
+
+
 def test_check_max_age_reverifies_corrupt_even_when_fresh(tmp_path, monkeypatch):
     # the age window only applies to intact records: a fresh corrupt record is still re-verified.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:

From ff6bb8b9da8d6f0ea6cf667cba975de1acf73508 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Mon, 3 Aug 2026 23:58:48 +0530
Subject: [PATCH 010/144] check: calendar-aware --max-age, symmetric clock-skew
 window

Parse --max-age with the calendar-aware relative time marker (m, y measured
against now), tolerate MAX_CLOCK_SKEW at both ends of the reuse window, and
correct the stale --repository-only comment.
---
 src/borg/archiver/check_cmd.py                | 28 ++++++++++++-------
 src/borg/repository.py                        | 15 ++++++----
 src/borg/testsuite/archiver/check_cmd_test.py |  2 +-
 src/borg/testsuite/repository_test.py         | 25 +++++++++++++++--
 4 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 333ccefa0b..825420e506 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -4,8 +4,9 @@
 from ..archive import ArchiveChecker
 from ..constants import *  # NOQA
 from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError
-from ..helpers import interval, yes, ArchiveFormatter
+from ..helpers import relative_time_marker_validator, yes, ArchiveFormatter
 from ..helpers.argparsing import ArgumentParser
+from ..helpers.time import archive_ts_now, calculate_relative_offset
 
 from ..logger import create_logger
 
@@ -45,6 +46,8 @@ def do_check(self, args, repository):
         if args.repair and args.max_duration:
             raise CommandError("--repair does not allow --max-duration argument.")
         if args.repair and args.max_age:
+            # reusing recorded results during repair depends on repository repair (refs #8572), which
+            # does not exist yet, so repair verifies every pack.
             raise CommandError("--repair does not allow the --max-age option.")
         if args.archives_only and args.max_age:
             # --max-age only affects the repository check, which --archives-only skips.
@@ -53,9 +56,8 @@ def do_check(self, args, repository):
             # partial checks progress by skipping packs whose record is younger than max_age.
             raise CommandError("--max-duration requires the --max-age option, e.g. --max-age=4w.")
         if args.max_duration and not args.repo_only:
-            # when doing a partial repo check, we can only do a low-level check of the repository files.
-            # archives check requires that a full repo check was done before and has built/cached a ChunkIndex.
-            # also, there is no max_duration support in the archives check code anyway.
+            # --max-duration time-boxes the repository pack check only; the archives check has no
+            # max_duration support and builds its own chunk index, so it cannot be split this way.
             raise CommandError("--repository-only is required for --max-duration support.")
         if not args.repo_only:
             # if we need the key later for the archives check, ask NOW for the passphrase! #1931
@@ -72,7 +74,13 @@ def do_check(self, args, repository):
             # the repository check has finished, which can take hours.
             ArchiveFormatter.validate_format(format)
         if not args.archives_only:
-            max_age = int(args.max_age.total_seconds()) if args.max_age else 0
+            if args.max_age:
+                # resolve the relative marker (e.g. 4w, 12m) to a concrete age in seconds; calendar
+                # units (m, y) are measured against now, like --older / --newer.
+                now = archive_ts_now()
+                max_age = int((now - calculate_relative_offset(args.max_age, now, earlier=True)).total_seconds())
+            else:
+                max_age = 0
             if not repository.check(repair=args.repair, max_duration=args.max_duration, max_age=max_age):
                 set_ec(EXIT_WARNING)
         if not args.repo_only and not archive_checker.check(
@@ -129,8 +137,8 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
 
         The ``--max-age`` option makes the check reuse the results of previous
         repository checks: packs whose intact result is younger than the given
-        interval (e.g. ``--max-age=4w``) are skipped, spreading the verification
-        cost over repeated checks. Check results are recorded in any case;
+        timespan (e.g. ``--max-age=4w`` or ``--max-age=12m``) are skipped, spreading
+        the verification cost over repeated checks. Check results are recorded in any case;
         ``--max-age`` only controls their reuse. Packs recorded corrupt are always
         re-verified. ``--max-age`` affects only the repository check and cannot be
         combined with ``--archives-only`` or ``--repair``.
@@ -242,12 +250,12 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         )
         subparser.add_argument(
             "--max-age",
-            metavar="INTERVAL",
+            metavar="TIMESPAN",
             dest="max_age",
-            type=interval,
+            type=relative_time_marker_validator,
             default=None,
             action=Highlander,
-            help="reuse intact-pack check results younger than INTERVAL (e.g. 4w)",
+            help="reuse intact-pack check results younger than TIMESPAN, e.g. 4w or 12m",
         )
         subparser.add_argument(
             "--max-duration",
diff --git a/src/borg/repository.py b/src/borg/repository.py
index fe3566edcf..b3f97b584e 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -841,7 +841,8 @@ def check(self, repair=False, max_duration=0, max_age=0):
         the pack, or when repair salvages and drops it (refs #8572).
 
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
-        max_age. Check results are always recorded and kept.
+        max_age, tolerating up to MAX_CLOCK_SKEW of clock skew at both ends of the window. Check
+        results are always recorded and kept.
         """
 
         def verify(namespace, name):
@@ -869,7 +870,7 @@ def store_list(namespace):
         if not len(tracker):
             logger.info("Starting from beginning.")
         elif max_age:
-            logger.info(f"{len(tracker)} pack check results on record, reusing those younger than max_age.")
+            logger.info(f"{len(tracker)} pack check results on record, reusing those younger than --max-age.")
         else:
             logger.info(f"{len(tracker)} pack check results on record, verifying every pack.")
         t_start = time.monotonic()
@@ -905,12 +906,14 @@ def store_list(namespace):
                 pack_pi.show(increase=1)  # advance for skipped packs too, so the bar tracks packs/, not work done
                 pack_id = hex_to_bin(info.name)
                 entry = tracker.get(pack_id)
-                # skip the pack if its recorded intact result is younger than max_age. a future
-                # timestamp (writer clock ahead of ours) also counts as recent, tolerated up to
-                # MAX_CLOCK_SKEW or max_age ahead, whichever is smaller.
+                # skip the pack if its recorded intact result is younger than max_age. the timestamp
+                # is written by whichever client ran the earlier check, so it may be off by up to
+                # MAX_CLOCK_SKEW against our clock in either direction; tolerate that much skew at both
+                # ends of the window, but never more than the window itself.
                 if entry is not None and entry.result and max_age:
                     age = time.time() - entry.timestamp
-                    if -min(MAX_CLOCK_SKEW, max_age) <= age < max_age:
+                    skew = min(MAX_CLOCK_SKEW, max_age)
+                    if -skew <= age <= max_age + skew:
                         pack_skipped += 1
                         continue
                 pack_files += 1
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index fb225f1d37..b3380202dd 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -95,7 +95,7 @@ def test_check_max_age(archivers, request):
     output = cmd(archiver, "check", "-v", "--repository-only", exit_code=0)
     assert "Starting full repository check" in output
     output = cmd(archiver, "check", "-v", "--repository-only", "--max-age=4w", exit_code=0)
-    assert "reusing those younger than max_age" in output
+    assert "reusing those younger than --max-age" in output
     assert "no problems found" in output
 
 
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 467722caca..bdac69d4b9 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1272,24 +1272,43 @@ def test_check_max_age_skips_fresh_ok(tmp_path, monkeypatch):
 
 
 def test_check_max_age_reverifies_stale_ok(tmp_path, monkeypatch):
-    # with max_age, a pack whose intact record is older than max_age is re-verified.
+    # with max_age, a pack whose intact record is older than max_age plus the skew tolerance is re-verified.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
+        max_age = 50
         tracker = PackTracker.new(repository.store)
-        old_ts = int(time.time()) - 100
+        old_ts = int(time.time()) - (max_age + MAX_CLOCK_SKEW + 100)  # clearly beyond max_age + skew
         tracker.table[intact_id] = PackTracker.Entry(timestamp=old_ts, result=1)
         tracker.save()
 
         hashed_keys = _spy_hash(repository, monkeypatch)
 
-        assert repository.check(repair=False, max_age=50) is True
+        assert repository.check(repair=False, max_age=max_age) is True
         assert pack_key in hashed_keys  # stale, re-verified
 
         after = PackTracker.load(repository.store)
         assert after.table[intact_id].timestamp > old_ts  # record refreshed
 
 
+def test_check_max_age_skips_stale_within_skew(tmp_path, monkeypatch):
+    # a record older than max_age but within MAX_CLOCK_SKEW of it still counts as recent (writer clock
+    # behind ours), so it is not re-verified.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, pack_key = _store_intact_pack(repository)
+
+        max_age = MAX_CLOCK_SKEW * 2  # wide window, so the skew tolerance is MAX_CLOCK_SKEW
+        tracker = PackTracker.new(repository.store)
+        past_ts = int(time.time()) - (max_age + MAX_CLOCK_SKEW // 2)  # past the window, within skew
+        tracker.table[intact_id] = PackTracker.Entry(timestamp=past_ts, result=1)
+        tracker.save()
+
+        hashed_keys = _spy_hash(repository, monkeypatch)
+
+        assert repository.check(repair=False, max_age=max_age) is True
+        assert pack_key not in hashed_keys  # within MAX_CLOCK_SKEW past the window, still recent
+
+
 def test_check_max_age_skips_near_future_ok(tmp_path, monkeypatch):
     # with a window wider than MAX_CLOCK_SKEW, a record timestamped up to MAX_CLOCK_SKEW into the
     # future (writer clock ahead of ours) still counts as recent and is not re-verified.

From 0138c35e01884ddb75d79bdd1fa1ca5cc1de9b9d Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 29 Jul 2026 15:26:36 +0530
Subject: [PATCH 011/144] check: report missing chunks inverted as chunk ->
 files -> archives, #9218

---
 src/borg/archive.py                           | 25 +++++++++++++++++--
 src/borg/testsuite/archiver/check_cmd_test.py | 24 ++++++++++++------
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 3299d07399..ff76238bf7 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -2130,6 +2130,9 @@ def rebuild_archives(
     ):
         """Analyze and rebuild archives, expecting some damage and trying to make stuff consistent again."""
 
+        missing_chunk_size: dict = {}  # chunk_id -> chunk size in bytes
+        missing_chunk_refs: defaultdict = defaultdict(lambda: defaultdict(set))  # chunk_id -> {path: {archive_name}}
+
         def add_callback(chunk):
             id_ = self.key.id_hash(chunk)
             cdata = self.repo_objs.format(id_, {}, chunk, ro_type=ROBJ_ARCHIVE_STREAM)
@@ -2146,16 +2149,22 @@ def add_reference(id_, size, cdata):
                     self.chunks.update_pack_info(pack_results)
 
         def verify_file_chunks(archive_name, item):
-            """Verifies that all file chunks are present. Missing file chunks will be logged."""
+            """Verify that all file chunks are present.
+
+            Record each missing chunk's size in missing_chunk_size and, in missing_chunk_refs, the
+            file path and archive it occurs in. Log each missing chunk at debug level.
+            """
             offset = 0
             for chunk in item.chunks:
                 chunk_id, size = chunk
                 if chunk_id not in self.chunks:
-                    logger.error(
+                    logger.debug(
                         "{}: {}: Missing file chunk detected (Byte {}-{}, Chunk {}).".format(
                             archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id)
                         )
                     )
+                    missing_chunk_size[chunk_id] = size
+                    missing_chunk_refs[chunk_id][item.path].add(archive_name)
                     self.error_found = True
                 offset += size
             if "size" in item:
@@ -2169,6 +2178,17 @@ def verify_file_chunks(archive_name, item):
                         )
                     )
 
+        def report_missing_chunks():
+            """Log the missing chunks, each with its size and the files and archives referencing it."""
+            if not missing_chunk_refs:
+                return
+            logger.error("The following chunks are missing in the repository:")
+            for chunk_id, refs in missing_chunk_refs.items():
+                logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {missing_chunk_size[chunk_id]:,} bytes")
+                for path in sorted(refs):
+                    archive_names = ", ".join(sorted(refs[path]))
+                    logger.error(f"    - {path}: {archive_names}")
+
         def robust_iterator(archive):
             """Iterates through all archive items
 
@@ -2326,6 +2346,7 @@ def valid_item(obj):
                 if archive_id != new_archive_id:
                     self.manifest.archives.delete_by_id(archive_id)
         pi.finish()
+        report_missing_chunks()
 
     def finish(self):
         if self.repair:
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index ef42ad776b..5766093893 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -178,9 +178,13 @@ def test_missing_file_chunk(archivers, request):
             pytest.fail("should not happen")  # convert 'fail'
 
     output = cmd(archiver, "check", exit_code=1)
-    assert "Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(killed_chunk.id) in output
+    assert src_file in output
     output = cmd(archiver, "check", "--repair", exit_code=0)
-    assert "Missing file chunk detected" in output  # repair is not changing anything, just reporting.
+    # repair is not changing anything, just reporting.
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(killed_chunk.id) in output
 
     # check does not modify the chunks list.
     for archive_name in ("archive1", "archive2"):
@@ -200,7 +204,7 @@ def test_missing_file_chunk(archivers, request):
 
     # check should not complain anymore about missing chunks:
     output = cmd(archiver, "check", "-v", "--repair", exit_code=0)
-    assert "Missing file chunk detected" not in output
+    assert "The following chunks are missing in the repository:" not in output
 
 
 def test_missing_archive_item_chunk(archivers, request):
@@ -483,11 +487,14 @@ def test_verify_data(archivers, request, init_args):
     # repair will find the defect chunk and remove it
     output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0)
     assert f"{bin_to_hex(chunk.id)}, integrity error" in output
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(chunk.id) in output
+    assert src_file in output
 
     # run with --verify-data again, it will notice the missing chunk.
     output = cmd(archiver, "check", "--archives-only", "--verify-data", exit_code=1)
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(chunk.id) in output
 
 
 def test_verify_data_wrong_chunk_content(archivers, request, monkeypatch):
@@ -571,12 +578,15 @@ def test_corrupted_file_chunk(archivers, request, init_args):
     # repair: the defect chunk will be removed.
     output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0)
     assert f"{bin_to_hex(chunk.id)}, integrity error" in output
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(chunk.id) in output
+    assert src_file in output
 
     # run normal check again
     cmd(archiver, "check", "--repository-only", exit_code=0)
     output = cmd(archiver, "check", "--archives-only", exit_code=1)
-    assert f"{src_file}: Missing file chunk detected" in output
+    assert "The following chunks are missing in the repository:" in output
+    assert src_file in output
 
 
 @pytest.mark.skip(

From 06f5889db4fd56162abe9ef65d26feeece29357a Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 30 Jul 2026 18:16:19 +0530
Subject: [PATCH 012/144] check: bound the missing chunks report memory and
 test its grouping, #9218

Cap distinct chunks and file refs kept for the end-of-run report, combine the two collection dicts into one, and add tests for the grouping and truncation.
---
 src/borg/archive.py                           | 50 ++++++++++++++-----
 src/borg/testsuite/archiver/check_cmd_test.py | 37 ++++++++++++--
 2 files changed, 71 insertions(+), 16 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index ff76238bf7..6e4cf01fbb 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -1849,6 +1849,11 @@ def __next__(self):
 
 
 class ArchiveChecker:
+    # Bound how many missing file chunks rebuild_archives buffers for its end-of-run report,
+    # so checking a badly damaged repo with very many missing chunks can not exhaust memory.
+    MAX_MISSING_CHUNKS = 10000  # max. distinct missing chunk ids kept for the report
+    MAX_REFS_PER_CHUNK = 100  # max. referencing files kept per missing chunk
+
     def __init__(self):
         self.error_found = False
         self.key = None
@@ -2130,8 +2135,28 @@ def rebuild_archives(
     ):
         """Analyze and rebuild archives, expecting some damage and trying to make stuff consistent again."""
 
-        missing_chunk_size: dict = {}  # chunk_id -> chunk size in bytes
-        missing_chunk_refs: defaultdict = defaultdict(lambda: defaultdict(set))  # chunk_id -> {path: {archive_name}}
+        # Missing file chunks, collected during the per-archive checks and reported grouped as
+        # chunk -> files -> archives after all archives were analyzed. Bounded by
+        # MAX_MISSING_CHUNKS / MAX_REFS_PER_CHUNK.
+        missing_chunks = {}  # chunk_id -> [size, {path: {archive_name}}]
+        missing_chunks_truncated = False  # True once the MAX_MISSING_CHUNKS cap was hit
+        missing_refs_truncated = set()  # chunk_ids whose MAX_REFS_PER_CHUNK cap was hit
+
+        def record_missing_chunk(archive_name, path, chunk_id, size):
+            nonlocal missing_chunks_truncated
+            entry = missing_chunks.get(chunk_id)
+            if entry is None:
+                if len(missing_chunks) >= self.MAX_MISSING_CHUNKS:
+                    missing_chunks_truncated = True
+                    return
+                entry = missing_chunks[chunk_id] = [size, {}]
+            refs = entry[1]
+            if path in refs:
+                refs[path].add(archive_name)
+            elif len(refs) < self.MAX_REFS_PER_CHUNK:
+                refs[path] = {archive_name}
+            else:
+                missing_refs_truncated.add(chunk_id)
 
         def add_callback(chunk):
             id_ = self.key.id_hash(chunk)
@@ -2149,11 +2174,7 @@ def add_reference(id_, size, cdata):
                     self.chunks.update_pack_info(pack_results)
 
         def verify_file_chunks(archive_name, item):
-            """Verify that all file chunks are present.
-
-            Record each missing chunk's size in missing_chunk_size and, in missing_chunk_refs, the
-            file path and archive it occurs in. Log each missing chunk at debug level.
-            """
+            """Verify that all of a file's chunks are present, collecting any missing ones for the report."""
             offset = 0
             for chunk in item.chunks:
                 chunk_id, size = chunk
@@ -2163,8 +2184,7 @@ def verify_file_chunks(archive_name, item):
                             archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id)
                         )
                     )
-                    missing_chunk_size[chunk_id] = size
-                    missing_chunk_refs[chunk_id][item.path].add(archive_name)
+                    record_missing_chunk(archive_name, item.path, chunk_id, size)
                     self.error_found = True
                 offset += size
             if "size" in item:
@@ -2179,15 +2199,19 @@ def verify_file_chunks(archive_name, item):
                     )
 
         def report_missing_chunks():
-            """Log the missing chunks, each with its size and the files and archives referencing it."""
-            if not missing_chunk_refs:
+            """Report the collected missing chunks, grouped as chunk -> files -> archives."""
+            if not missing_chunks:
                 return
             logger.error("The following chunks are missing in the repository:")
-            for chunk_id, refs in missing_chunk_refs.items():
-                logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {missing_chunk_size[chunk_id]:,} bytes")
+            for chunk_id, (size, refs) in missing_chunks.items():
+                logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {size:,} bytes")
                 for path in sorted(refs):
                     archive_names = ", ".join(sorted(refs[path]))
                     logger.error(f"    - {path}: {archive_names}")
+                if chunk_id in missing_refs_truncated:
+                    logger.error(f"    - ... (only the first {self.MAX_REFS_PER_CHUNK} files are listed)")
+            if missing_chunks_truncated:
+                logger.error(f"... (only the first {self.MAX_MISSING_CHUNKS} missing chunks are listed)")
 
         def robust_iterator(archive):
             """Iterates through all archive items
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 5766093893..444df71a55 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -6,7 +6,7 @@
 
 import pytest
 
-from ...archive import ChunkBuffer
+from ...archive import ChunkBuffer, ArchiveChecker
 from ...constants import *  # NOQA
 from ...helpers import bin_to_hex, msgpack, CommandError, IntegrityError
 from ...manifest import Manifest
@@ -179,8 +179,12 @@ def test_missing_file_chunk(archivers, request):
 
     output = cmd(archiver, "check", exit_code=1)
     assert "The following chunks are missing in the repository:" in output
-    assert bin_to_hex(killed_chunk.id) in output
-    assert src_file in output
+    # archive1 and archive2 share src_file, so the missing chunk appears once, with both archives
+    # listed on its single reference line.
+    assert output.count(bin_to_hex(killed_chunk.id)) == 1
+    ref_lines = [line for line in output.splitlines() if src_file in line]
+    assert len(ref_lines) == 1
+    assert "archive1" in ref_lines[0] and "archive2" in ref_lines[0]
     output = cmd(archiver, "check", "--repair", exit_code=0)
     # repair is not changing anything, just reporting.
     assert "The following chunks are missing in the repository:" in output
@@ -207,6 +211,33 @@ def test_missing_file_chunk(archivers, request):
     assert "The following chunks are missing in the repository:" not in output
 
 
+def test_missing_file_chunk_report_truncated(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+
+    # remove several distinct file chunks, so more missing chunks exist than the (patched) report limit.
+    archive, repository = open_archive(archiver.repository_path, "archive1")
+    killed_ids = []
+    with repository:
+        for item in archive.iter_items():
+            if "chunks" not in item or not item.chunks:
+                continue
+            chunk_id = item.chunks[-1].id
+            if chunk_id not in killed_ids:
+                repository.delete(chunk_id)
+                killed_ids.append(chunk_id)
+            if len(killed_ids) >= 3:
+                break
+    assert len(killed_ids) >= 2  # need several distinct missing chunks to exercise truncation
+
+    # cap the report to a single chunk, so the remaining missing chunks are truncated.
+    with patch.object(ArchiveChecker, "MAX_MISSING_CHUNKS", 1):
+        output = cmd(archiver, "check", exit_code=1)
+    assert "The following chunks are missing in the repository:" in output
+    assert output.count("- Chunk ") == 1  # only one chunk is detailed
+    assert "only the first 1 missing chunks are listed" in output  # the rest are noted as truncated
+
+
 def test_missing_archive_item_chunk(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)

From 36b29672e4ee40957f343b0a2e149cf0a32ff047 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Tue, 4 Aug 2026 00:34:35 +0530
Subject: [PATCH 013/144] check: use format_file_size for missing chunk report
 and test the per-chunk refs cap, #9218

---
 src/borg/archive.py                           |  2 +-
 src/borg/testsuite/archiver/check_cmd_test.py | 29 +++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 6e4cf01fbb..bbab274445 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -2204,7 +2204,7 @@ def report_missing_chunks():
                 return
             logger.error("The following chunks are missing in the repository:")
             for chunk_id, (size, refs) in missing_chunks.items():
-                logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {size:,} bytes")
+                logger.error(f"- Chunk {bin_to_hex(chunk_id)}, {format_file_size(size)}")
                 for path in sorted(refs):
                     archive_names = ", ".join(sorted(refs[path]))
                     logger.error(f"    - {path}: {archive_names}")
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 444df71a55..086df5f3f4 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -16,6 +16,7 @@
     cmd,
     src_file,
     create_src_archive,
+    create_regular_file,
     open_archive,
     generate_archiver_tests,
     read_chunk,
@@ -238,6 +239,34 @@ def test_missing_file_chunk_report_truncated(archivers, request):
     assert "only the first 1 missing chunks are listed" in output  # the rest are noted as truncated
 
 
+def test_missing_file_chunk_refs_truncated(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+
+    # several distinct files with identical content dedup to the same chunk, so a single missing
+    # chunk ends up referenced by many files, which exercises the per-chunk reference cap.
+    for i in range(5):
+        create_regular_file(archiver.input_path, f"samefile{i}", contents=b"same content for dedup")
+    cmd(archiver, "create", "archive1", "input")
+
+    archive, repository = open_archive(archiver.repository_path, "archive1")
+    killed_id = None
+    with repository:
+        for item in archive.iter_items():
+            if item.path.endswith("samefile0"):
+                killed_id = item.chunks[0].id
+                repository.delete(killed_id)
+                break
+    assert killed_id is not None
+
+    # cap references per chunk to 2, so the remaining referencing files are truncated.
+    with patch.object(ArchiveChecker, "MAX_REFS_PER_CHUNK", 2):
+        output = cmd(archiver, "check", exit_code=1)
+    assert "The following chunks are missing in the repository:" in output
+    assert bin_to_hex(killed_id) in output
+    assert "only the first 2 files are listed" in output  # the remaining referencing files are truncated
+
+
 def test_missing_archive_item_chunk(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)

From 63ab760e2c877295a10643660f23724676b4ac3a Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 5 Aug 2026 00:45:55 +0530
Subject: [PATCH 014/144] check: run truncation tests binary-safe, report
 missing-ref total, use a tuple for missing chunk entries, #9218

---
 src/borg/archive.py                           | 15 ++++++++++-----
 src/borg/testsuite/archiver/check_cmd_test.py | 19 ++++++++++---------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index bbab274445..8e9fbb3e62 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -2138,19 +2138,21 @@ def rebuild_archives(
         # Missing file chunks, collected during the per-archive checks and reported grouped as
         # chunk -> files -> archives after all archives were analyzed. Bounded by
         # MAX_MISSING_CHUNKS / MAX_REFS_PER_CHUNK.
-        missing_chunks = {}  # chunk_id -> [size, {path: {archive_name}}]
+        missing_chunks = {}  # chunk_id -> (size, {path: {archive_name}})
         missing_chunks_truncated = False  # True once the MAX_MISSING_CHUNKS cap was hit
         missing_refs_truncated = set()  # chunk_ids whose MAX_REFS_PER_CHUNK cap was hit
+        missing_refs_total = 0  # total missing chunk references seen (every file x chunk occurrence, uncapped)
 
         def record_missing_chunk(archive_name, path, chunk_id, size):
-            nonlocal missing_chunks_truncated
+            nonlocal missing_chunks_truncated, missing_refs_total
+            missing_refs_total += 1
             entry = missing_chunks.get(chunk_id)
             if entry is None:
                 if len(missing_chunks) >= self.MAX_MISSING_CHUNKS:
                     missing_chunks_truncated = True
                     return
-                entry = missing_chunks[chunk_id] = [size, {}]
-            refs = entry[1]
+                entry = missing_chunks[chunk_id] = (size, {})
+            size, refs = entry
             if path in refs:
                 refs[path].add(archive_name)
             elif len(refs) < self.MAX_REFS_PER_CHUNK:
@@ -2211,7 +2213,10 @@ def report_missing_chunks():
                 if chunk_id in missing_refs_truncated:
                     logger.error(f"    - ... (only the first {self.MAX_REFS_PER_CHUNK} files are listed)")
             if missing_chunks_truncated:
-                logger.error(f"... (only the first {self.MAX_MISSING_CHUNKS} missing chunks are listed)")
+                logger.error(
+                    f"... (only the first {self.MAX_MISSING_CHUNKS} missing chunks are listed; "
+                    f"{missing_refs_total} missing chunk references total)"
+                )
 
         def robust_iterator(archive):
             """Iterates through all archive items
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 086df5f3f4..a55b81d783 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -212,8 +212,9 @@ def test_missing_file_chunk(archivers, request):
     assert "The following chunks are missing in the repository:" not in output
 
 
-def test_missing_file_chunk_report_truncated(archivers, request):
-    archiver = request.getfixturevalue(archivers)
+def test_missing_file_chunk_report_truncated(archiver):
+    # local-only: this patches ArchiveChecker.MAX_MISSING_CHUNKS in-process, which has no effect
+    # when borg runs as a separate process (binary_archiver), so it must not be parametrized.
     check_cmd_setup(archiver)
 
     # remove several distinct file chunks, so more missing chunks exist than the (patched) report limit.
@@ -243,9 +244,11 @@ def test_missing_file_chunk_refs_truncated(archivers, request):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
 
-    # several distinct files with identical content dedup to the same chunk, so a single missing
-    # chunk ends up referenced by many files, which exercises the per-chunk reference cap.
-    for i in range(5):
+    # many distinct files with identical content dedup to the same chunk, so a single missing chunk
+    # ends up referenced by more files than MAX_REFS_PER_CHUNK, which exercises the per-chunk cap
+    # without patching (so it works in binary mode too, where borg runs as a separate process).
+    cap = ArchiveChecker.MAX_REFS_PER_CHUNK
+    for i in range(cap + 1):
         create_regular_file(archiver.input_path, f"samefile{i}", contents=b"same content for dedup")
     cmd(archiver, "create", "archive1", "input")
 
@@ -259,12 +262,10 @@ def test_missing_file_chunk_refs_truncated(archivers, request):
                 break
     assert killed_id is not None
 
-    # cap references per chunk to 2, so the remaining referencing files are truncated.
-    with patch.object(ArchiveChecker, "MAX_REFS_PER_CHUNK", 2):
-        output = cmd(archiver, "check", exit_code=1)
+    output = cmd(archiver, "check", exit_code=1)
     assert "The following chunks are missing in the repository:" in output
     assert bin_to_hex(killed_id) in output
-    assert "only the first 2 files are listed" in output  # the remaining referencing files are truncated
+    assert f"only the first {cap} files are listed" in output  # the remaining referencing files are truncated
 
 
 def test_missing_archive_item_chunk(archivers, request):

From 112cbca449444e7da98c4032e3dec4e262537e63 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 5 Aug 2026 01:01:54 +0530
Subject: [PATCH 015/144] check: stream one line per missing chunk id, run
 report on abort, lower report caps, #9218

---
 src/borg/archive.py                           | 123 +++++++++---------
 src/borg/testsuite/archiver/check_cmd_test.py |   9 +-
 2 files changed, 71 insertions(+), 61 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 8e9fbb3e62..271118ac86 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -1851,8 +1851,8 @@ def __next__(self):
 class ArchiveChecker:
     # Bound how many missing file chunks rebuild_archives buffers for its end-of-run report,
     # so checking a badly damaged repo with very many missing chunks can not exhaust memory.
-    MAX_MISSING_CHUNKS = 10000  # max. distinct missing chunk ids kept for the report
-    MAX_REFS_PER_CHUNK = 100  # max. referencing files kept per missing chunk
+    MAX_MISSING_CHUNKS = 1000  # max. distinct missing chunk ids kept for the grouped report
+    MAX_REFS_PER_CHUNK = 10  # max. referencing files kept per missing chunk
 
     def __init__(self):
         self.error_found = False
@@ -2152,6 +2152,8 @@ def record_missing_chunk(archive_name, path, chunk_id, size):
                     missing_chunks_truncated = True
                     return
                 entry = missing_chunks[chunk_id] = (size, {})
+                # one line per chunk id (not per file), so an interrupted check still logs what it found.
+                logger.error(f"Missing chunk detected: {bin_to_hex(chunk_id)}, {format_file_size(size)}.")
             size, refs = entry
             if path in refs:
                 refs[path].add(archive_name)
@@ -2319,63 +2321,68 @@ def valid_item(obj):
         pi = ProgressIndicatorPercent(
             total=num_archives, msg="Checking archives %3.1f%%", step=0.1, msgid="check.rebuild_archives"
         )
-        for i, info in enumerate(archive_infos):
-            pi.show(i)
-            archive_id, archive_id_hex = info.id, bin_to_hex(info.id)
-            try:
-                formatted = formatter.format_item(info, jsonline=False)
-            except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityErrorBase):
-                # keys like {comment} need the archive metadata, which is damaged or missing here.
-                # use the values from the archive directory entry, they are always available.
-                formatted = f"{info.name} {OutputTimestamp(info.ts)} {archive_id_hex}"
-            logger.info(f"Analyzing archive {formatted} ({i + 1}/{num_archives})")
-            if archive_id not in self.chunks:
-                logger.error(f"Archive metadata block {archive_id_hex} is missing!")
-                self.error_found = True
-                if self.repair:
-                    logger.error(f"Deleting broken archive {info.name} {archive_id_hex}.")
-                    self.manifest.archives.delete_by_id(archive_id)
-                else:
-                    logger.error(f"Would delete broken archive {info.name} {archive_id_hex}.")
-                continue
-            cdata = self.repository.get(archive_id)
-            try:
-                _, data = self.repo_objs.parse(archive_id, cdata, ro_type=ROBJ_ARCHIVE_META)
-            except IntegrityErrorBase as integrity_error:
-                logger.error(f"Archive metadata block {archive_id_hex} is corrupted: {integrity_error}")
-                self.error_found = True
+        # report the missing chunks collected so far even if the loop is interrupted (Ctrl-C) or aborts
+        # with an exception (e.g. the "Unknown archive metadata version" raise below), so a check of a
+        # badly damaged repo does not throw away everything it already found.
+        try:
+            for i, info in enumerate(archive_infos):
+                pi.show(i)
+                archive_id, archive_id_hex = info.id, bin_to_hex(info.id)
+                try:
+                    formatted = formatter.format_item(info, jsonline=False)
+                except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityErrorBase):
+                    # keys like {comment} need the archive metadata, which is damaged or missing here.
+                    # use the values from the archive directory entry, they are always available.
+                    formatted = f"{info.name} {OutputTimestamp(info.ts)} {archive_id_hex}"
+                logger.info(f"Analyzing archive {formatted} ({i + 1}/{num_archives})")
+                if archive_id not in self.chunks:
+                    logger.error(f"Archive metadata block {archive_id_hex} is missing!")
+                    self.error_found = True
+                    if self.repair:
+                        logger.error(f"Deleting broken archive {info.name} {archive_id_hex}.")
+                        self.manifest.archives.delete_by_id(archive_id)
+                    else:
+                        logger.error(f"Would delete broken archive {info.name} {archive_id_hex}.")
+                    continue
+                cdata = self.repository.get(archive_id)
+                try:
+                    _, data = self.repo_objs.parse(archive_id, cdata, ro_type=ROBJ_ARCHIVE_META)
+                except IntegrityErrorBase as integrity_error:
+                    logger.error(f"Archive metadata block {archive_id_hex} is corrupted: {integrity_error}")
+                    self.error_found = True
+                    if self.repair:
+                        logger.error(f"Deleting broken archive {info.name} {archive_id_hex}.")
+                        self.manifest.archives.delete_by_id(archive_id)
+                    else:
+                        logger.error(f"Would delete broken archive {info.name} {archive_id_hex}.")
+                    continue
+                archive = self.key.unpack_archive(data)
+                archive = ArchiveItem(internal_dict=archive)
+                if archive.version != 2:
+                    raise Exception("Unknown archive metadata version")
+                items_buffer = ChunkBuffer(self.key)
+                items_buffer.write_chunk = add_callback
+                for item in robust_iterator(archive):
+                    if "chunks" in item:
+                        verify_file_chunks(info.name, item)
+                    items_buffer.add(item)
+                items_buffer.flush(flush=True)
                 if self.repair:
-                    logger.error(f"Deleting broken archive {info.name} {archive_id_hex}.")
-                    self.manifest.archives.delete_by_id(archive_id)
-                else:
-                    logger.error(f"Would delete broken archive {info.name} {archive_id_hex}.")
-                continue
-            archive = self.key.unpack_archive(data)
-            archive = ArchiveItem(internal_dict=archive)
-            if archive.version != 2:
-                raise Exception("Unknown archive metadata version")
-            items_buffer = ChunkBuffer(self.key)
-            items_buffer.write_chunk = add_callback
-            for item in robust_iterator(archive):
-                if "chunks" in item:
-                    verify_file_chunks(info.name, item)
-                items_buffer.add(item)
-            items_buffer.flush(flush=True)
-            if self.repair:
-                archive.item_ptrs = archive_put_items(
-                    items_buffer.chunks, repo_objs=self.repo_objs, add_reference=add_reference
-                )
-                data = self.key.pack_metadata(archive.as_dict())
-                new_archive_id = self.key.id_hash(data)
-                logger.debug(f"archive id old: {bin_to_hex(archive_id)}")
-                logger.debug(f"archive id new: {bin_to_hex(new_archive_id)}")
-                cdata = self.repo_objs.format(new_archive_id, {}, data, ro_type=ROBJ_ARCHIVE_META)
-                add_reference(new_archive_id, len(data), cdata)
-                self.manifest.archives.create(info.name, new_archive_id, info.ts)
-                if archive_id != new_archive_id:
-                    self.manifest.archives.delete_by_id(archive_id)
-        pi.finish()
-        report_missing_chunks()
+                    archive.item_ptrs = archive_put_items(
+                        items_buffer.chunks, repo_objs=self.repo_objs, add_reference=add_reference
+                    )
+                    data = self.key.pack_metadata(archive.as_dict())
+                    new_archive_id = self.key.id_hash(data)
+                    logger.debug(f"archive id old: {bin_to_hex(archive_id)}")
+                    logger.debug(f"archive id new: {bin_to_hex(new_archive_id)}")
+                    cdata = self.repo_objs.format(new_archive_id, {}, data, ro_type=ROBJ_ARCHIVE_META)
+                    add_reference(new_archive_id, len(data), cdata)
+                    self.manifest.archives.create(info.name, new_archive_id, info.ts)
+                    if archive_id != new_archive_id:
+                        self.manifest.archives.delete_by_id(archive_id)
+        finally:
+            pi.finish()
+            report_missing_chunks()
 
     def finish(self):
         if self.repair:
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index a55b81d783..64c3fc990f 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -180,9 +180,12 @@ def test_missing_file_chunk(archivers, request):
 
     output = cmd(archiver, "check", exit_code=1)
     assert "The following chunks are missing in the repository:" in output
-    # archive1 and archive2 share src_file, so the missing chunk appears once, with both archives
-    # listed on its single reference line.
-    assert output.count(bin_to_hex(killed_chunk.id)) == 1
+    # archive1 and archive2 share src_file, so the missing chunk is grouped once, with both archives
+    # listed on its single reference line (the id also appears once in the streamed "Missing chunk
+    # detected" line emitted while the archives are analyzed).
+    killed_hex = bin_to_hex(killed_chunk.id)
+    chunk_header_lines = [ln for ln in output.splitlines() if ln.startswith("- Chunk ") and killed_hex in ln]
+    assert len(chunk_header_lines) == 1
     ref_lines = [line for line in output.splitlines() if src_file in line]
     assert len(ref_lines) == 1
     assert "archive1" in ref_lines[0] and "archive2" in ref_lines[0]

From 23e2a210c3a06efffd8b1b8776b0e3d6c534bc56 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 5 Aug 2026 01:31:40 +0530
Subject: [PATCH 016/144] check: re-verify records past max-age, guard on
 resolved window, tighten comments

---
 src/borg/archiver/check_cmd.py                | 34 ++++++++++---------
 src/borg/repository.py                        | 24 ++++++-------
 src/borg/testsuite/archiver/check_cmd_test.py |  8 +++++
 src/borg/testsuite/repository_test.py         | 27 +++++++--------
 4 files changed, 49 insertions(+), 44 deletions(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 825420e506..6e37e61b16 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -43,21 +43,30 @@ def do_check(self, args, repository):
             )
         if args.repo_only and args.find_lost_archives:
             raise CommandError("--repository-only contradicts the --find-lost-archives option.")
+        # resolve the marker (e.g. 4w, 12m) to seconds; calendar units (m, y) count from now, as for
+        # --older/--newer. the validator returns a string (truthy even for a zero span like 0d), so
+        # the --max-duration guard below tests max_age, the resolved value, not args.max_age.
+        if args.max_age is not None:
+            now = archive_ts_now()
+            max_age = int((now - calculate_relative_offset(args.max_age, now, earlier=True)).total_seconds())
+        else:
+            max_age = 0
         if args.repair and args.max_duration:
             raise CommandError("--repair does not allow --max-duration argument.")
-        if args.repair and args.max_age:
-            # reusing recorded results during repair depends on repository repair (refs #8572), which
-            # does not exist yet, so repair verifies every pack.
+        if args.repair and args.max_age is not None:
+            # repair verifies every pack; reusing recorded results during repair needs repository
+            # repair (refs #8572).
             raise CommandError("--repair does not allow the --max-age option.")
-        if args.archives_only and args.max_age:
-            # --max-age only affects the repository check, which --archives-only skips.
+        if args.archives_only and args.max_age is not None:
+            # --max-age only affects the repository check; --archives-only skips it.
             raise CommandError("--archives-only does not allow the --max-age option.")
-        if args.max_duration and not args.max_age:
-            # partial checks progress by skipping packs whose record is younger than max_age.
+        if args.max_duration and not max_age:
+            # a partial check advances by skipping packs younger than max_age, so it needs a nonzero
+            # window (--max-age=0d resolves to 0).
             raise CommandError("--max-duration requires the --max-age option, e.g. --max-age=4w.")
         if args.max_duration and not args.repo_only:
-            # --max-duration time-boxes the repository pack check only; the archives check has no
-            # max_duration support and builds its own chunk index, so it cannot be split this way.
+            # --max-duration limits only the repository check; the archives check has no max_duration
+            # support.
             raise CommandError("--repository-only is required for --max-duration support.")
         if not args.repo_only:
             # if we need the key later for the archives check, ask NOW for the passphrase! #1931
@@ -74,13 +83,6 @@ def do_check(self, args, repository):
             # the repository check has finished, which can take hours.
             ArchiveFormatter.validate_format(format)
         if not args.archives_only:
-            if args.max_age:
-                # resolve the relative marker (e.g. 4w, 12m) to a concrete age in seconds; calendar
-                # units (m, y) are measured against now, like --older / --newer.
-                now = archive_ts_now()
-                max_age = int((now - calculate_relative_offset(args.max_age, now, earlier=True)).total_seconds())
-            else:
-                max_age = 0
             if not repository.check(repair=args.repair, max_duration=args.max_duration, max_age=max_age):
                 set_ec(EXIT_WARNING)
         if not args.repo_only and not archive_checker.check(
diff --git a/src/borg/repository.py b/src/borg/repository.py
index b3f97b584e..4d38a2c822 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -836,13 +836,13 @@ def check(self, repair=False, max_duration=0, max_age=0):
         corrupt packs and dropping those packs is left to repair, refs #8572. The ids of the packs
         found corrupt are kept in cache/checked-packs for repair, refs #9696.
 
-        Any pack on record as corrupt fails the check, including on a partial run that stops before
-        re-reaching it. The record clears when the pack verifies intact again, when compact removes
-        the pack, or when repair salvages and drops it (refs #8572).
+        A pack recorded corrupt fails the check, also on a partial run that stops before re-reaching
+        it. The record clears at the check that finds the pack intact again or gone (removed by
+        compact, or salvaged and dropped by repair; refs #8572); prune() does this from packs/.
 
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
-        max_age, tolerating up to MAX_CLOCK_SKEW of clock skew at both ends of the window. Check
-        results are always recorded and kept.
+        max_age, accepting a future timestamp up to MAX_CLOCK_SKEW (clock skew). Results are recorded
+        regardless of max_age.
         """
 
         def verify(namespace, name):
@@ -906,14 +906,12 @@ def store_list(namespace):
                 pack_pi.show(increase=1)  # advance for skipped packs too, so the bar tracks packs/, not work done
                 pack_id = hex_to_bin(info.name)
                 entry = tracker.get(pack_id)
-                # skip the pack if its recorded intact result is younger than max_age. the timestamp
-                # is written by whichever client ran the earlier check, so it may be off by up to
-                # MAX_CLOCK_SKEW against our clock in either direction; tolerate that much skew at both
-                # ends of the window, but never more than the window itself.
+                # skip a pack recorded intact within the last max_age seconds. the timestamp is set
+                # by the client that ran the earlier check; accept a future one (negative age) up to
+                # MAX_CLOCK_SKEW, and re-verify anything at or past max_age.
                 if entry is not None and entry.result and max_age:
                     age = time.time() - entry.timestamp
-                    skew = min(MAX_CLOCK_SKEW, max_age)
-                    if -skew <= age <= max_age + skew:
+                    if -min(MAX_CLOCK_SKEW, max_age) <= age < max_age:
                         pack_skipped += 1
                         continue
                 pack_files += 1
@@ -947,8 +945,8 @@ def store_list(namespace):
         if pack_skipped:
             summary += f" Reused {pack_skipped} recent pack check result(s)."
         logger.info(summary)
-        # every pack recorded corrupt, not only those verified this run; empty if the index is
-        # corrupt, since then no packs were scanned.
+        # corrupt_ids() is every pack recorded corrupt, including from earlier runs. with a corrupt
+        # index the packs were not scanned, so report nothing.
         corrupt_ids = tracker.corrupt_ids() if index_errors == 0 else []
         if corrupt_ids:
             # one id per line (the list can be long).
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index b3380202dd..b4bad53dc4 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -79,17 +79,25 @@ def test_check_max_age(archivers, request):
     check_cmd_setup(archiver)
 
     # --repair and --archives-only do not allow --max-age, and --max-duration requires --max-age.
+    # a zero span like 0d resolves to max_age=0 (no reuse), so it must not satisfy the guards either.
     if archiver.FORK_DEFAULT:
         cmd(archiver, "check", "--repair", "--max-age=1d", exit_code=CommandError().exit_code)
+        cmd(archiver, "check", "--repair", "--max-age=0d", exit_code=CommandError().exit_code)
         cmd(archiver, "check", "--archives-only", "--max-age=1d", exit_code=CommandError().exit_code)
         cmd(archiver, "check", "--repository-only", "--max-duration=3600", exit_code=CommandError().exit_code)
+        ec = CommandError().exit_code
+        cmd(archiver, "check", "--repository-only", "--max-duration=3600", "--max-age=0d", exit_code=ec)
     else:
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--repair", "--max-age=1d")
+        with pytest.raises(CommandError):
+            cmd(archiver, "check", "--repair", "--max-age=0d")
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--archives-only", "--max-age=1d")
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--repository-only", "--max-duration=3600")
+        with pytest.raises(CommandError):
+            cmd(archiver, "check", "--repository-only", "--max-duration=3600", "--max-age=0d")
 
     # a check records its results, a later one with --max-age reuses them.
     output = cmd(archiver, "check", "-v", "--repository-only", exit_code=0)
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index bdac69d4b9..80983e1dca 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1058,7 +1058,7 @@ def test_check_partial_rechecks_pack_sorting_before_checked_one(tmp_path):
 
 
 def test_check_partial_rechecks_pack_recorded_corrupt(tmp_path):
-    # a pack recorded corrupt earlier is re-verified, so the corruption keeps being reported.
+    # a pack recorded corrupt earlier is re-verified and reported again.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         corrupt_id = H(1)  # stored content does not hash to this name
         repository.store_store("packs/" + bin_to_hex(corrupt_id), b"CORRUPT-does-not-match-name")
@@ -1272,13 +1272,13 @@ def test_check_max_age_skips_fresh_ok(tmp_path, monkeypatch):
 
 
 def test_check_max_age_reverifies_stale_ok(tmp_path, monkeypatch):
-    # with max_age, a pack whose intact record is older than max_age plus the skew tolerance is re-verified.
+    # with max_age, a pack whose intact record is older than max_age is re-verified.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
         max_age = 50
         tracker = PackTracker.new(repository.store)
-        old_ts = int(time.time()) - (max_age + MAX_CLOCK_SKEW + 100)  # clearly beyond max_age + skew
+        old_ts = int(time.time()) - (max_age + 100)  # clearly beyond max_age
         tracker.table[intact_id] = PackTracker.Entry(timestamp=old_ts, result=1)
         tracker.save()
 
@@ -1291,27 +1291,25 @@ def test_check_max_age_reverifies_stale_ok(tmp_path, monkeypatch):
         assert after.table[intact_id].timestamp > old_ts  # record refreshed
 
 
-def test_check_max_age_skips_stale_within_skew(tmp_path, monkeypatch):
-    # a record older than max_age but within MAX_CLOCK_SKEW of it still counts as recent (writer clock
-    # behind ours), so it is not re-verified.
+def test_check_max_age_reverifies_stale_within_skew(tmp_path, monkeypatch):
+    # a record past max_age gets no skew tolerance: it is re-verified even within MAX_CLOCK_SKEW of it.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
-        max_age = MAX_CLOCK_SKEW * 2  # wide window, so the skew tolerance is MAX_CLOCK_SKEW
+        max_age = MAX_CLOCK_SKEW * 2  # window wider than MAX_CLOCK_SKEW
         tracker = PackTracker.new(repository.store)
-        past_ts = int(time.time()) - (max_age + MAX_CLOCK_SKEW // 2)  # past the window, within skew
+        past_ts = int(time.time()) - (max_age + MAX_CLOCK_SKEW // 2)  # just past the window
         tracker.table[intact_id] = PackTracker.Entry(timestamp=past_ts, result=1)
         tracker.save()
 
         hashed_keys = _spy_hash(repository, monkeypatch)
 
         assert repository.check(repair=False, max_age=max_age) is True
-        assert pack_key not in hashed_keys  # within MAX_CLOCK_SKEW past the window, still recent
+        assert pack_key in hashed_keys  # past max_age, re-verified
 
 
 def test_check_max_age_skips_near_future_ok(tmp_path, monkeypatch):
-    # with a window wider than MAX_CLOCK_SKEW, a record timestamped up to MAX_CLOCK_SKEW into the
-    # future (writer clock ahead of ours) still counts as recent and is not re-verified.
+    # a future timestamp (writer clock ahead of ours) up to MAX_CLOCK_SKEW is tolerated and skipped.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
@@ -1323,12 +1321,11 @@ def test_check_max_age_skips_near_future_ok(tmp_path, monkeypatch):
         hashed_keys = _spy_hash(repository, monkeypatch)
 
         assert repository.check(repair=False, max_age=MAX_CLOCK_SKEW * 2) is True
-        assert pack_key not in hashed_keys  # within MAX_CLOCK_SKEW ahead, still counts as recent
+        assert pack_key not in hashed_keys  # within MAX_CLOCK_SKEW ahead, skipped
 
 
 def test_check_max_age_reverifies_far_future_ok(tmp_path, monkeypatch):
-    # with a window wider than MAX_CLOCK_SKEW, a record more than MAX_CLOCK_SKEW into the future is
-    # not plausible clock skew and is re-verified.
+    # a future timestamp more than MAX_CLOCK_SKEW ahead exceeds the skew tolerance and is re-verified.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         intact_id, pack_key = _store_intact_pack(repository)
 
@@ -1340,7 +1337,7 @@ def test_check_max_age_reverifies_far_future_ok(tmp_path, monkeypatch):
         hashed_keys = _spy_hash(repository, monkeypatch)
 
         assert repository.check(repair=False, max_age=MAX_CLOCK_SKEW * 2) is True
-        assert pack_key in hashed_keys  # too far ahead to be clock skew, re-verified
+        assert pack_key in hashed_keys  # more than MAX_CLOCK_SKEW ahead, re-verified
 
 
 def test_check_max_age_reverifies_future_beyond_small_window(tmp_path, monkeypatch):

From 7171b688b8b38ee1090a26c2d2da2e7a2f86435a Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 5 Aug 2026 01:46:36 +0530
Subject: [PATCH 017/144] check: verify least-recently-checked packs first on
 partial runs, document --max-age markers

---
 src/borg/archiver/check_cmd.py        | 11 +++++---
 src/borg/repository.py                |  9 +++++++
 src/borg/testsuite/repository_test.py | 39 +++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 6e37e61b16..4118825170 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -140,10 +140,13 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         The ``--max-age`` option makes the check reuse the results of previous
         repository checks: packs whose intact result is younger than the given
         timespan (e.g. ``--max-age=4w`` or ``--max-age=12m``) are skipped, spreading
-        the verification cost over repeated checks. Check results are recorded in any case;
-        ``--max-age`` only controls their reuse. Packs recorded corrupt are always
-        re-verified. ``--max-age`` affects only the repository check and cannot be
-        combined with ``--archives-only`` or ``--repair``.
+        the verification cost over repeated checks. The timespan uses the same markers
+        as ``--older``/``--newer``: ``d``, ``w``, ``H``, ``M``, ``S`` are exact spans,
+        while ``m`` and ``y`` are calendar units counted from now (so ``12m`` equals
+        ``1y``). Check results are recorded in any case; ``--max-age`` only controls
+        their reuse. Packs recorded corrupt are always re-verified. ``--max-age``
+        affects only the repository check and cannot be combined with
+        ``--archives-only`` or ``--repair``.
 
         The ``--max-duration`` option can be used to split a long-running repository
         check into multiple partial checks. After the given number of seconds, the
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 4d38a2c822..6c3683b640 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -900,6 +900,15 @@ def store_list(namespace):
         if index_errors == 0:
             # packs are the bulk of the work and the part --max-duration spreads over several checks.
             pack_infos = store_list("packs")
+            if partial:
+                # a partial check stops after max_duration; verify the least-recently-checked packs
+                # first so repeated runs cover every pack. sort by recorded check time, unrecorded
+                # (time 0) first.
+                def recorded_ts(info):
+                    entry = tracker.get(hex_to_bin(info.name))
+                    return entry.timestamp if entry is not None else 0
+
+                pack_infos.sort(key=recorded_ts)
             pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs")
             for info in pack_infos:
                 self._lock_refresh()
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 80983e1dca..8ca323ebab 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1414,6 +1414,45 @@ def test_check_max_age_partial_progress(tmp_path, monkeypatch):
         assert pack_a_id in after.table and pack_b_id in after.table
 
 
+def test_check_partial_verifies_least_recently_checked_first(tmp_path, monkeypatch):
+    # a partial check verifies the least-recently-checked packs first. with a budget for one pack, the
+    # pack with the oldest recorded result is verified, though its id sorts last.
+    max_age = 3600
+    recent_id = b"\x00" * 32  # sorts first by id
+    older_id = b"\xff" * 32  # sorts last by id
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        for pid in (recent_id, older_id):
+            repository.store_store("packs/" + bin_to_hex(pid), b"content-" + pid[:4])
+
+        now = int(time.time())
+        tracker = PackTracker.new(repository.store)
+        # both records are past max_age, so both need re-verification; the more recently recorded pack
+        # has the id that sorts first.
+        tracker.table[recent_id] = PackTracker.Entry(timestamp=now - (max_age + 100), result=1)
+        tracker.table[older_id] = PackTracker.Entry(timestamp=now - (max_age + 100000), result=1)
+        tracker.save()
+
+        # freeze the clock and jump it past max_duration after the first hash, so exactly one pack is
+        # verified this run.
+        clock = {"t": 0.0}
+        monkeypatch.setattr(time, "monotonic", lambda: clock["t"])
+        hashed_keys = []
+        orig_hash = repository.store.hash
+
+        def hash_and_advance(key):
+            hashed_keys.append(key)
+            result = orig_hash(key)
+            clock["t"] = 10**9
+            return result
+
+        monkeypatch.setattr(repository.store, "hash", hash_and_advance)
+
+        repository.check(repair=False, max_duration=1, max_age=max_age)
+
+        pack_hashes = [key for key in hashed_keys if key.startswith("packs/")]
+        assert pack_hashes == ["packs/" + bin_to_hex(older_id)]  # the oldest recorded pack
+
+
 def test_check_max_age_reuses_records_of_plain_check(tmp_path, monkeypatch):
     # a check without max_age records its results, a later check with max_age reuses them.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:

From 3a8431ccd80406bd8631ef3ebfe64c60137cc972 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 1 Aug 2026 03:05:12 +0200
Subject: [PATCH 018/144] completion: generate fish completions, remove
 hand-written ones, fixes #9989

fish completions are now generated by shtab, like the bash, zsh and tcsh
ones, extended with a fish preamble that provides the same dynamic
completions the bash/zsh scripts have (archive names, aid: archive IDs,
tags, sort keys, files cache modes, compression specs, chunker params,
relative time markers, timestamps, file sizes, help topics).

This needs shtab >= 1.9.3, which brings the fixed fish backend (see
tqdm/shtab#227, #228, #229, #230 and #231), so the pin was bumped.

Arguments taking a filesystem path are now typed, so that all shells can
complete them: such a path was either untyped or just a str, so the
shells relied on their default completion (bash/zsh/tcsh) or offered
nothing at all (fish). FilesystemPathSpec is used for arguments taking a
file (--exclude-from, --patterns-from, TARFILE, the borg debug file
arguments, borg key export/import PATH), the new FilesystemDirSpec for
arguments taking a directory (MOUNTPOINT, borg benchmark crud PATH).
Besides the completions, they now also reject empty paths and slashify
them on Windows, like the other filesystem path arguments do.
Note that borg key export/import PATH completed directories before
(it takes a file) and borg benchmark crud PATH completed files (it takes
a directory) - both are fixed by this.

shtab's fish FILE completion needs a little help to also complete
absolute paths, see tqdm/shtab#245 (fix: tqdm/shtab#246).

The completion script is now always generated for the "borg" command,
no matter how borg was invoked while generating it - completions
generated by "python -m borg completion ..." were useless before, as
the shells would complete a "__main__.py" command.

The hand-written, unmaintained fish completions in
scripts/shell_completions/ are removed, as are their tests.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 pyproject.toml                                |   2 +-
 scripts/shell_completions/fish/borg.fish      | 545 ------------------
 src/borg/archiver/_common.py                  |   3 +
 src/borg/archiver/benchmark_cmd.py            |   5 +-
 src/borg/archiver/completion_cmd.py           | 247 ++++++--
 src/borg/archiver/debug_cmd.py                |  44 +-
 src/borg/archiver/key_cmds.py                 |  12 +-
 src/borg/archiver/mount_cmds.py               |   8 +-
 src/borg/archiver/tar_cmds.py                 |  12 +-
 src/borg/helpers/__init__.py                  |   1 +
 src/borg/helpers/parseformat.py               |   9 +
 .../testsuite/archiver/completion_cmd_test.py | 205 +++++++
 src/borg/testsuite/shell_completions_test.py  |  22 -
 13 files changed, 483 insertions(+), 632 deletions(-)
 delete mode 100644 scripts/shell_completions/fish/borg.fish
 delete mode 100644 src/borg/testsuite/shell_completions_test.py

diff --git a/pyproject.toml b/pyproject.toml
index 0b6e90d5db..74322d99e0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,7 +37,7 @@ dependencies = [
   "platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'",  # for macOS: breaking changes in 3.0.0.
   "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'",  # for others: 2.6+ works consistently.
   "argon2-cffi",
-  "shtab>=1.8.0,!=1.8.2,!=1.9.0,!=1.9.1",  # these generate broken zsh completions, see tqdm/shtab#224
+  "shtab>=1.9.3",  # older versions generate broken zsh (tqdm/shtab#224) or fish (tqdm/shtab#227) completions
   "backports-zstd; python_version < '3.14'", # for python < 3.14.
   "jsonargparse>=4.47.0",
   "PyYAML>=6.0.2",  # we need to register our types with yaml, jsonargparse uses yaml for config files
diff --git a/scripts/shell_completions/fish/borg.fish b/scripts/shell_completions/fish/borg.fish
deleted file mode 100644
index a756e08940..0000000000
--- a/scripts/shell_completions/fish/borg.fish
+++ /dev/null
@@ -1,545 +0,0 @@
-# Completions for borg
-# https://www.borgbackup.org/
-# Note:
-# Listing archives works on password-protected repositories only if $BORG_PASSPHRASE is set.
-# Install:
-# Copy this file to /usr/share/fish/vendor_completions.d/
-
-# Commands
-
-complete -c borg -f -n __fish_is_first_token -a 'analyze' -d 'Analyze archives to find "hot spots"'
-complete -c borg -f -n __fish_is_first_token -a 'create' -d 'Create a new archive'
-complete -c borg -f -n __fish_is_first_token -a 'extract' -d 'Extract archive contents'
-complete -c borg -f -n __fish_is_first_token -a 'check' -d 'Check repository consistency'
-complete -c borg -f -n __fish_is_first_token -a 'rename' -d 'Rename an existing archive'
-complete -c borg -f -n __fish_is_first_token -a 'list' -d 'List archive or repository contents'
-complete -c borg -f -n __fish_is_first_token -a 'diff' -d 'Find differences between archives'
-complete -c borg -f -n __fish_is_first_token -a 'delete' -d 'Delete an archive'
-complete -c borg -f -n __fish_is_first_token -a 'prune' -d 'Prune repository archives'
-complete -c borg -f -n __fish_is_first_token -a 'compact' -d 'Free repository space'
-complete -c borg -f -n __fish_is_first_token -a 'info' -d 'Show archive details'
-complete -c borg -f -n __fish_is_first_token -a 'mount' -d 'Mount archive or a repository'
-complete -c borg -f -n __fish_is_first_token -a 'umount' -d 'Unmount the mounted archive'
-complete -c borg -f -n __fish_is_first_token -a 'repo-compress' -d 'Repository (re-)compression'
-complete -c borg -f -n __fish_is_first_token -a 'repo-create' -d 'Create a new, empty repository'
-complete -c borg -f -n __fish_is_first_token -a 'repo-delete' -d 'Delete a repository'
-complete -c borg -f -n __fish_is_first_token -a 'repo-info' -d 'Show repository information'
-complete -c borg -f -n __fish_is_first_token -a 'repo-list' -d 'List repository contents'
-complete -c borg -f -n __fish_is_first_token -a 'repo-space' -d 'Manage reserved space in a repository'
-complete -c borg -f -n __fish_is_first_token -a 'tag' -d 'Tag archives'
-complete -c borg -f -n __fish_is_first_token -a 'transfer' -d 'Transfer of archives from another repository'
-complete -c borg -f -n __fish_is_first_token -a 'undelete' -d 'Undelete archives'
-complete -c borg -f -n __fish_is_first_token -a 'version' -d 'Display Borg client version / Borg server version'
-
-function __fish_borg_seen_key
-    if __fish_seen_subcommand_from key
-        and not __fish_seen_subcommand_from import export change-passphrase
-        return 0
-    end
-    return 1
-end
-complete -c borg -f -n __fish_is_first_token -a 'key' -d 'Manage a repository key'
-complete -c borg -f -n __fish_borg_seen_key  -a 'import' -d 'Import a repository key'
-complete -c borg -f -n __fish_borg_seen_key  -a 'export' -d 'Export a repository key'
-complete -c borg -f -n __fish_borg_seen_key  -a 'change-passphrase' -d 'Change key file passphrase'
-
-complete -c borg -f -n __fish_is_first_token -a 'serve' -d 'Start in server mode'
-complete -c borg -f -n __fish_is_first_token -a 'recreate' -d 'Recreate contents of existing archives'
-complete -c borg -f -n __fish_is_first_token -a 'export-tar' -d 'Create tarball from an archive'
-complete -c borg -f -n __fish_is_first_token -a 'with-lock' -d 'Run a command while the repository lock is held'
-complete -c borg -f -n __fish_is_first_token -a 'break-lock' -d 'Break the repository lock'
-
-function __fish_borg_seen_benchmark
-    if __fish_seen_subcommand_from benchmark
-        and not __fish_seen_subcommand_from crud
-        return 0
-    end
-    return 1
-end
-complete -c borg -f -n __fish_is_first_token -a 'benchmark' -d 'Benchmark borg operations'
-complete -c borg -f -n __fish_borg_seen_benchmark -a 'crud' -d 'Benchmark borg CRUD operations'
-
-function __fish_borg_seen_help
-    if __fish_seen_subcommand_from help
-        and not __fish_seen_subcommand_from patterns placeholders compression
-        return 0
-    end
-    return 1
-end
-complete -c borg -f -n __fish_is_first_token -a 'help' -d 'Miscellaneous Help'
-complete -c borg -f -n __fish_borg_seen_help -a 'patterns' -d 'Help for patterns'
-complete -c borg -f -n __fish_borg_seen_help -a 'placeholders' -d 'Help for placeholders'
-complete -c borg -f -n __fish_borg_seen_help -a 'compression' -d 'Help for compression'
-
-# Common options
-complete -c borg -f -s h -l 'help'                  -d 'Show help information'
-complete -c borg -f      -l 'version'               -d 'Show version information'
-complete -c borg -f      -l 'critical'              -d 'Log level CRITICAL'
-complete -c borg -f      -l 'error'                 -d 'Log level ERROR'
-complete -c borg -f      -l 'warning'               -d 'Log level WARNING (default)'
-complete -c borg -f      -l 'info'                  -d 'Log level INFO'
-complete -c borg -f -s v -l 'verbose'               -d 'Log level INFO'
-complete -c borg -f      -l 'debug'                 -d 'Log level DEBUG'
-complete -c borg -f      -l 'debug-topic'           -d 'Enable TOPIC debugging'
-complete -c borg -f -s p -l 'progress'              -d 'Show progress information'
-complete -c borg -f      -l 'log-json'              -d 'Output one JSON object per log line'
-complete -c borg -f      -l 'lock-wait'             -d 'Wait for lock max N seconds [1]'
-complete -c borg -f      -l 'show-version'          -d 'Log version information'
-complete -c borg -f      -l 'show-rc'               -d 'Log the return code'
-complete -c borg -f      -l 'umask'                 -d 'Set umask to M [0077]'
-
-# borg analyze options
-complete -c borg -f -s a -l 'match-archives'        -d 'Only archive names matching PATTERN'        -n "__fish_seen_subcommand_from analyze"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from analyze"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from analyze"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from analyze"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from analyze"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from analyze"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from analyze"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from analyze"
-
-# borg repo-compress options
-# Define compression methods once at the top
-set -l compression_methods "none auto lz4 zstd,1 zstd,2 zstd,3 zstd,4 zstd,5 zstd,6 zstd,7 zstd,8 zstd,9 zstd,10 zstd,11 zstd,12 zstd,13 zstd,14 zstd,15 zstd,16 zstd,17 zstd,18 zstd,19 zstd,20 zstd,21 zstd,22 zlib,1 zlib,2 zlib,3 zlib,4 zlib,5 zlib,6 zlib,7 zlib,8 zlib,9 lzma,0 lzma,1 lzma,2 lzma,3 lzma,4 lzma,5 lzma,6 lzma,7 lzma,8 lzma,9"
-complete -c borg -f -s C -l 'compression'           -d 'Select compression ALGORITHM,LEVEL [lz4]' -a "$compression_methods" -n "__fish_seen_subcommand_from repo-compress"
-complete -c borg -f -s s -l 'stats'                 -d 'Print statistics'                            -n "__fish_seen_subcommand_from repo-compress"
-
-# borg create options
-complete -c borg -f -s n -l 'dry-run'               -d 'Do not create a backup archive'                 -n "__fish_seen_subcommand_from create"
-complete -c borg -f -s s -l 'stats'                 -d 'Print verbose statistics'                       -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'list'                  -d 'Print verbose list of items'                    -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'filter'                -d 'Only items with given STATUSCHARS'              -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'json'                  -d 'Print verbose stats as json'                    -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'stdin-name'            -d 'Use NAME in archive for stdin data'             -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'content-from-command'  -d 'Interpret PATH as command and store its stdout' -n "__fish_seen_subcommand_from create"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'                 -n "__fish_seen_subcommand_from create"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'         -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'pattern'               -d 'Include/exclude paths matching PATTERN'         -n "__fish_seen_subcommand_from create"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'         -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'exclude-caches'        -d 'Exclude directories tagged as cache'            -n "__fish_seen_subcommand_from create"
-complete -c borg         -l 'exclude-if-present'    -d 'Exclude directories that contain FILENAME'      -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'keep-exclude-tags'     -d 'Keep tag files of excluded directories'         -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'exclude-nodump'        -d 'Exclude files flagged NODUMP'                   -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'exclude-dataless'      -d 'Exclude files flagged DATALESS (macOS)'         -n "__fish_seen_subcommand_from create"
-# Filesystem options
-complete -c borg -f -s x -l 'one-file-system'       -d 'Stay in the same file system'                   -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'numeric-ids'           -d 'Only store numeric user:group identifiers'      -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'noatime'               -d 'Do not store atime'                             -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'noctime'               -d 'Do not store ctime'                             -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'nobirthtime'           -d 'Do not store creation date'                     -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'nobsdflags'            -d 'Do not store bsdflags'                          -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'noacls'                -d 'Do not read and store ACLs into archive'        -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'noxattrs'              -d 'Do not read and store xattrs into archive'      -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'noflags'               -d 'Do not store flags'                             -n "__fish_seen_subcommand_from create"
-set -l files_cache_mode "ctime,size,inode mtime,size,inode ctime,size mtime,size rechunk,ctime rechunk,mtime size disabled"
-complete -c borg -f      -l 'files-cache'           -d 'Operate files cache in MODE' -a "$files_cache_mode" -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'read-special'          -d 'Open device files like regular files'           -n "__fish_seen_subcommand_from create"
-# Archive options
-complete -c borg -f      -l 'comment'               -d 'Add COMMENT to the archive'                     -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'timestamp'             -d 'Set creation TIME (yyyy-mm-ddThh:mm:ss)'        -n "__fish_seen_subcommand_from create"
-complete -c borg         -l 'timestamp'             -d 'Set creation time by reference FILE'            -n "__fish_seen_subcommand_from create"
-complete -c borg -f      -l 'chunker-params'        -d 'Chunker PARAMETERS [19,23,21,4095]'             -n "__fish_seen_subcommand_from create"
-complete -c borg -f -s C -l 'compression'           -d 'Select compression ALGORITHM,LEVEL [lz4]' -a "$compression_methods" -n "__fish_seen_subcommand_from create"
-
-# borg extract options
-complete -c borg -f      -l 'list'                  -d 'Print verbose list of items'                -n "__fish_seen_subcommand_from extract"
-complete -c borg -f -s n -l 'dry-run'               -d 'Do not actually extract any files'          -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'numeric-ids'           -d 'Only obey numeric user:group identifiers'   -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'nobsdflags'            -d 'Do not extract/set bsdflags'                -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'noflags'               -d 'Do not extract/set flags'                   -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'noacls'                -d 'Do not extract/set ACLs'                    -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'noxattrs'              -d 'Do not extract/set xattrs'                  -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'stdout'                -d 'Write all extracted data to stdout'         -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'sparse'                -d 'Create holes in output sparse file'         -n "__fish_seen_subcommand_from extract"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'             -n "__fish_seen_subcommand_from extract"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'     -n "__fish_seen_subcommand_from extract"
-complete -c borg         -l 'pattern'               -d 'Include/exclude paths matching PATTERN'     -n "__fish_seen_subcommand_from extract"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'     -n "__fish_seen_subcommand_from extract"
-complete -c borg -f      -l 'strip-components'      -d 'Remove NUMBER of leading path elements'     -n "__fish_seen_subcommand_from extract"
-
-# borg check options
-complete -c borg -f      -l 'repository-only'       -d 'Only perform repository checks'             -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'archives-only'         -d 'Only perform archives checks'               -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'verify-data'           -d 'Cryptographic integrity verification'       -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'repair'                -d 'Attempt to repair found inconsistencies'    -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'max-duration'          -d 'Partial repo check for max. SECONDS'        -n "__fish_seen_subcommand_from check"
-# Archive filters
-complete -c borg -f -s P -l 'prefix'                -d 'Only archive names starting with PREFIX'    -n "__fish_seen_subcommand_from check"
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from check"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from check"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from check"
-
-# borg rename
-# no specific options
-
-# borg list options
-complete -c borg -f      -l 'short'                 -d 'Only print file/directory names'            -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'format'                -d 'Specify FORMAT for file listing'            -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'json'                  -d 'List contents in json format'               -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'json-lines'            -d 'List contents in json lines format'         -n "__fish_seen_subcommand_from list"
-# Archive filters
-complete -c borg -f -s P -l 'prefix'                -d 'Only archive names starting with PREFIX'    -n "__fish_seen_subcommand_from list"
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from list"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from list"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'             -n "__fish_seen_subcommand_from list"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'     -n "__fish_seen_subcommand_from list"
-complete -c borg -f      -l 'pattern'               -d 'Include/exclude paths matching PATTERN'     -n "__fish_seen_subcommand_from list"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'     -n "__fish_seen_subcommand_from list"
-
-# borg diff options
-complete -c borg -f      -l 'numeric-ids'           -d 'Only consider numeric user:group'           -n "__fish_seen_subcommand_from diff"
-complete -c borg -f      -l 'same-chunker-params'   -d 'Override check of chunker parameters'       -n "__fish_seen_subcommand_from diff"
-complete -c borg -f      -l 'sort'                  -d 'Sort the output lines by file path'         -n "__fish_seen_subcommand_from diff"
-complete -c borg -f      -l 'json-lines'            -d 'Format output as JSON Lines'                -n "__fish_seen_subcommand_from diff"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'             -n "__fish_seen_subcommand_from diff"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'     -n "__fish_seen_subcommand_from diff"
-complete -c borg -f      -l 'pattern'               -d 'Include/exclude paths matching PATTERN'     -n "__fish_seen_subcommand_from diff"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'     -n "__fish_seen_subcommand_from diff"
-
-# borg delete options
-complete -c borg -f -s n -l 'dry-run'               -d 'Do not change the repository'               -n "__fish_seen_subcommand_from delete"
-complete -c borg -f      -l 'list'                  -d 'Output verbose list of archives'              -n "__fish_seen_subcommand_from delete"
-# Archive filters
-complete -c borg -f -s P -l 'prefix'                -d 'Only archive names starting with PREFIX'    -n "__fish_seen_subcommand_from delete"
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from delete"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from delete"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from delete"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from delete"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from delete"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from delete"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from delete"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from delete"
-
-# borg prune options
-complete -c borg -f -s n -l 'dry-run'               -d 'Do not change the repository'               -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'force'                 -d 'Force pruning of corrupted archives'        -n "__fish_seen_subcommand_from prune"
-complete -c borg -f -s s -l 'stats'                 -d 'Print verbose statistics'                   -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'list'                  -d 'Print verbose list of items'                -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'keep-within'           -d 'Keep archives within time INTERVAL'         -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'keep-last'             -d 'NUMBER of secondly archives to keep'        -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'keep-secondly'         -d 'NUMBER of secondly archives to keep'        -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'keep-minutely'         -d 'NUMBER of minutely archives to keep'        -n "__fish_seen_subcommand_from prune"
-complete -c borg -f -s H -l 'keep-hourly'           -d 'NUMBER of hourly archives to keep'          -n "__fish_seen_subcommand_from prune"
-complete -c borg -f -s d -l 'keep-daily'            -d 'NUMBER of daily archives to keep'           -n "__fish_seen_subcommand_from prune"
-complete -c borg -f -s w -l 'keep-weekly'           -d 'NUMBER of weekly archives to keep'          -n "__fish_seen_subcommand_from prune"
-complete -c borg -f -s m -l 'keep-monthly'          -d 'NUMBER of monthly archives to keep'         -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'keep-13weekly'         -d 'NUMBER of quarterly archives to keep (13 week strategy)' -n "__fish_seen_subcommand_from prune"
-complete -c borg -f      -l 'keep-3monthly'         -d 'NUMBER of quarterly archives to keep (3 month strategy)' -n "__fish_seen_subcommand_from prune"
-complete -c borg -f -s y -l 'keep-yearly'           -d 'NUMBER of yearly archives to keep'          -n "__fish_seen_subcommand_from prune"
-# Archive filters
-complete -c borg -f -s P -l 'prefix'                -d 'Only archive names starting with PREFIX'    -n "__fish_seen_subcommand_from prune"
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from prune"
-
-# borg compact options
-complete -c borg -f -s n -l 'dry-run'               -d 'Do nothing'                                  -n "__fish_seen_subcommand_from compact"
-complete -c borg -f -s s -l 'stats'                 -d 'Print statistics (might be much slower)'     -n "__fish_seen_subcommand_from compact"
-
-# borg info options
-complete -c borg -f      -l 'json'                  -d 'Format output in json format'               -n "__fish_seen_subcommand_from info"
-# Archive filters
-complete -c borg -f -s P -l 'prefix'                -d 'Only archive names starting with PREFIX'    -n "__fish_seen_subcommand_from info"
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from info"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from info"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from info"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from info"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from info"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from info"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from info"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from info"
-
-# borg repo-list options
-complete -c borg -f      -l 'short'                 -d 'Only print the archive IDs, nothing else'   -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'format'                -d 'Specify format for archive listing'         -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'json'                  -d 'Format output as JSON'                      -n "__fish_seen_subcommand_from repo-list"
-# Archive filters
-complete -c borg -f -s P -l 'prefix'                -d 'Only archive names starting with PREFIX'    -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from repo-list"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from repo-list"
-complete -c borg -f      -l 'deleted'               -d 'Consider only soft-deleted archives'        -n "__fish_seen_subcommand_from repo-list"
-
-# borg mount options
-complete -c borg -f -s f -l 'foreground'            -d 'Stay in foreground, do not daemonize'       -n "__fish_seen_subcommand_from mount"
-# FIXME This list is probably not full, but I tried to pick only those that are relevant to borg mount -o:
-set -l fuse_options "ac_attr_timeout= allow_damaged_files allow_other allow_root attr_timeout= auto auto_cache auto_unmount default_permissions entry_timeout= gid= group_id= kernel_cache max_read= negative_timeout= noauto noforget remember= remount rootmode= uid= umask= user user_id= versions"
-complete -c borg -f -s o                            -d 'Fuse mount OPTION' -a "$fuse_options"       -n "__fish_seen_subcommand_from mount"
-# Archive filters
-complete -c borg -f -s P -l 'prefix'                -d 'Only archive names starting with PREFIX'    -n "__fish_seen_subcommand_from mount"
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from mount"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from mount"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'             -n "__fish_seen_subcommand_from mount"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'     -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'pattern'               -d 'Include/exclude paths matching PATTERN'     -n "__fish_seen_subcommand_from mount"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'     -n "__fish_seen_subcommand_from mount"
-complete -c borg -f      -l 'strip-components'      -d 'Remove NUMBER of leading path elements'     -n "__fish_seen_subcommand_from mount"
-
-# borg umount
-# no specific options
-
-# borg tag options
-complete -c borg -f      -l 'set'                   -d 'Set tags (can be given multiple times)'      -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'add'                   -d 'Add tags (can be given multiple times)'      -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'remove'                -d 'Remove tags (can be given multiple times)'   -n "__fish_seen_subcommand_from tag"
-# Archive filters
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from tag"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from tag"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from tag"
-
-# borg key change-passphrase
-# no specific options
-
-# borg key export
-complete -c borg -f      -l 'paper'                 -d 'Create an export for printing'              -n "__fish_seen_subcommand_from export"
-complete -c borg -f      -l 'qr-html'               -d 'Create an html file for printing and qr'    -n "__fish_seen_subcommand_from export"
-
-# borg transfer options
-complete -c borg -f -s n -l 'dry-run'               -d 'Do not change repository, just check'       -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'other-repo'            -d 'Transfer archives from the other repository' -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'from-borg1'            -d 'Other repository is borg 1.x'               -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'upgrader'              -d 'Use the upgrader to convert transferred data' -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f -s C -l 'compression'           -d 'Select compression algorithm' -a "$compression_methods" -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'recompress'            -d 'Recompress chunks CONDITION' -a "$recompress_when" -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'chunker-params'        -d 'Chunker PARAMETERS [19,23,21,4095]'         -n "__fish_seen_subcommand_from transfer"
-# Archive filters
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from transfer"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from transfer"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from transfer"
-
-# borg key import
-complete -c borg -f      -l 'paper'                 -d 'Import from a backup done with --paper'     -n "__fish_seen_subcommand_from import"
-
-# borg undelete options
-complete -c borg -f -s n -l 'dry-run'               -d 'Do not change repository'                   -n "__fish_seen_subcommand_from undelete"
-complete -c borg -f      -l 'list'                  -d 'Output verbose list of archives'            -n "__fish_seen_subcommand_from undelete"
-# Archive filters
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from undelete"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from undelete"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from undelete"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from undelete"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from undelete"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from undelete"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from undelete"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from undelete"
-
-
-# borg recreate
-complete -c borg -f      -l 'list'                  -d 'Print verbose list of items'                -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'filter'                -d 'Only items with given STATUSCHARS'          -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f -s n -l 'dry-run'               -d 'Do not change the repository'               -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f -s s -l 'stats'                 -d 'Print verbose statistics'                   -n "__fish_seen_subcommand_from recreate"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'             -n "__fish_seen_subcommand_from recreate"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'     -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'pattern'               -d 'Include/exclude paths matching PATTERN'     -n "__fish_seen_subcommand_from recreate"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'     -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'exclude-caches'        -d 'Exclude directories tagged as cache'        -n "__fish_seen_subcommand_from recreate"
-complete -c borg         -l 'exclude-if-present'    -d 'Exclude directories that contain FILENAME'  -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'keep-exclude-tags'     -d 'Keep tag files of excluded directories'     -n "__fish_seen_subcommand_from recreate"
-# Archive filters
-complete -c borg -f -s a -l 'match-archives'        -d 'Only consider archives matching all patterns' -n "__fish_seen_subcommand_from recreate"
-set -l sort_keys "timestamp archive name id tags host user"
-complete -c borg -f      -l 'sort-by'               -d 'Sorting KEYS [timestamp]' -a "$sort_keys"   -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'first'                 -d 'Only first N archives'                      -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'last'                  -d 'Only last N archives'                       -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'oldest'                -d 'Consider archives within TIMESPAN from oldest' -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'newest'                -d 'Consider archives within TIMESPAN from newest' -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'older'                 -d 'Consider archives older than TIMESPAN'      -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'newer'                 -d 'Consider archives newer than TIMESPAN'      -n "__fish_seen_subcommand_from recreate"
-# Archive options
-complete -c borg -f      -l 'target'                -d "Create a new ARCHIVE"                       -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'comment'               -d 'Add COMMENT to the archive'                 -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'timestamp'             -d 'Set creation TIME (yyyy-mm-ddThh:mm:ss)'    -n "__fish_seen_subcommand_from recreate"
-complete -c borg         -l 'timestamp'             -d 'Set creation time using reference FILE'     -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f -s C -l 'compression'           -d 'Select compression ALGORITHM,LEVEL [lz4]' -a "$compression_methods" -n "__fish_seen_subcommand_from recreate"
-set -l recompress_when "if-different always never"
-complete -c borg -f      -l 'recompress'            -d 'Recompress chunks CONDITION' -a "$recompress_when" -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'chunker-params'        -d 'Chunker PARAMETERS [19,23,21,4095]'         -n "__fish_seen_subcommand_from recreate"
-
-# borg export-tar options
-complete -c borg         -l 'tar-filter'            -d 'Filter program to pipe data through'        -n "__fish_seen_subcommand_from export-tar"
-complete -c borg -f      -l 'list'                  -d 'Print verbose list of items'                -n "__fish_seen_subcommand_from export-tar"
-complete -c borg -f      -l 'tar-format'            -d 'Select tar format: BORG, PAX or GNU'        -n "__fish_seen_subcommand_from export-tar"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'             -n "__fish_seen_subcommand_from export-tar"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'     -n "__fish_seen_subcommand_from export-tar"
-complete -c borg -f      -l 'pattern'               -d 'Include/exclude paths matching PATTERN'     -n "__fish_seen_subcommand_from export-tar"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'     -n "__fish_seen_subcommand_from export-tar"
-complete -c borg -f      -l 'strip-components'      -d 'Remove NUMBER of leading path elements'     -n "__fish_seen_subcommand_from export-tar"
-
-# borg import-tar options
-complete -c borg         -l 'tar-filter'            -d 'Filter program to pipe data through'        -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f -s s -l 'stats'                 -d 'Print statistics for the created archive'   -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f      -l 'list'                  -d 'Print verbose list of items'                -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f      -l 'filter'                -d 'Only display items with given STATUSCHARS'  -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f      -l 'json'                  -d 'Output stats as JSON'                       -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f      -l 'ignore-zeros'          -d 'Ignore zero-filled blocks in the input'     -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f      -l 'comment'               -d 'Add COMMENT to the archive'                 -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f      -l 'timestamp'             -d 'Set creation TIME (yyyy-mm-ddThh:mm:ss)'    -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f      -l 'chunker-params'        -d 'Chunker PARAMETERS [19,23,21,4095]'         -n "__fish_seen_subcommand_from import-tar"
-complete -c borg -f -s C -l 'compression'           -d 'Select compression ALGORITHM,LEVEL [lz4]' -a "$compression_methods" -n "__fish_seen_subcommand_from import-tar"
-# Exclusion options
-complete -c borg    -s e -l 'exclude'               -d 'Exclude paths matching PATTERN'             -n "__fish_seen_subcommand_from recreate"
-complete -c borg         -l 'exclude-from'          -d 'Read exclude patterns from EXCLUDEFILE'     -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'pattern'               -d 'Include/exclude paths matching PATTERN'     -n "__fish_seen_subcommand_from recreate"
-complete -c borg         -l 'patterns-from'         -d 'Include/exclude paths from PATTERNFILE'     -n "__fish_seen_subcommand_from recreate"
-complete -c borg -f      -l 'strip-components'      -d 'Remove NUMBER of leading path elements'     -n "__fish_seen_subcommand_from recreate"
-
-# borg serve
-complete -c borg         -l 'restrict-to-path'      -d 'Restrict repository access to PATH'         -n "__fish_seen_subcommand_from serve"
-complete -c borg         -l 'restrict-to-repository' -d 'Restrict repository access at PATH'        -n "__fish_seen_subcommand_from serve"
-
-
-# borg with-lock
-# no specific options
-
-# borg break-lock
-# no specific options
-
-# borg benchmark
-# no specific options
-
-# borg help
-# no specific options
-
-
-function __fish_borg_archives
-    # additionally to aid:XXXXXXXX we show archive (series) name and timestamp
-    borg repo-list --format="aid:{id:.8}{TAB}{archive} {start}{NEWLINE}" 2>/dev/null
-end
-
-function __fish_borg_archive_arg --description 'Test if current command is a specific borg command with token count' --argument command token_count
-    # Check if we're in the context of a specific borg command
-    set -l tokens (commandline --tokenize)
-    set -l cmdline (commandline --current-process)
-
-    # Make sure we're in a borg command context
-    if not test $tokens[1] = "borg"
-        return 1
-    end
-
-    # Make sure we're in the specific command context
-    if not test $tokens[2] = "$command"
-        return 1
-    end
-
-    # Check if we're at the right token position
-    if not test (count $tokens) "-eq" "$token_count"
-        return 1
-    end
-
-    # Additional check to ensure we're not in the middle of typing an option
-    if string match --quiet --regex -- "^-" (commandline --current-token)
-        return 1
-    end
-
-    return 0
-end
-
-# The following completions use the -F flag to force disable filename completion
-# for various borg commands, ensuring only archive names are suggested.
-# We also use the -e flag to explicitly erase all default completions before adding our custom ones.
-
-# Global rules to disable filename completions for specific borg commands
-# This ensures that no filename completions are shown for these commands
-complete -c borg -e -n '__fish_seen_subcommand_from diff delete list info extract mount export-tar rename tag undelete recreate transfer check analyze'
-
-# First, explicitly erase all default completions for each command
-# This is the most specific rule and should take precedence
-complete -c borg -e -n '__fish_borg_archive_arg "diff" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "diff" 3'
-complete -c borg -e -n '__fish_borg_archive_arg "delete" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "list" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "info" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "extract" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "mount" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "export-tar" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "rename" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "tag" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "undelete" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "recreate" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "transfer" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "check" 2'
-complete -c borg -e -n '__fish_borg_archive_arg "analyze" 2'
-
-# Also add specific rules to disable filename completions at the exact position
-# This ensures that no filename completions are shown at the position where we expect archive names
-complete -c borg --no-files -n '__fish_borg_archive_arg "diff" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "diff" 3'
-complete -c borg --no-files -n '__fish_borg_archive_arg "delete" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "list" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "info" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "extract" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "mount" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "export-tar" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "rename" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "tag" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "undelete" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "recreate" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "transfer" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "check" 2'
-complete -c borg --no-files -n '__fish_borg_archive_arg "analyze" 2'
-
-# Then add our custom completions with high priority and no-files
-# This ensures no filename completions are shown and our custom completions take precedence
-complete -c borg -p 100 -n '__fish_borg_archive_arg "diff" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "diff" 3' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "delete" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "list" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "info" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "extract" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "mount" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "export-tar" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "rename" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "tag" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "undelete" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "recreate" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "transfer" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "check" 2' -a '(__fish_borg_archives)' --no-files
-complete -c borg -p 100 -n '__fish_borg_archive_arg "analyze" 2' -a '(__fish_borg_archives)' --no-files
diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py
index 77787c26a5..a07cb30bfb 100644
--- a/src/borg/archiver/_common.py
+++ b/src/borg/archiver/_common.py
@@ -8,6 +8,7 @@
 from ..cache import Cache, assert_secure
 from ..helpers import Error
 from ..helpers import SortBySpec, location_validator, Location, relative_time_marker_validator
+from ..helpers import FilesystemPathSpec
 from ..helpers import Highlander, octal_int
 from ..helpers.argparsing import SUPPRESS, PositiveInt
 from ..helpers.nanorst import rst_to_terminal
@@ -313,6 +314,7 @@ def define_exclude_and_patterns(add_option, *, tag_files=False, strip_components
     add_option(
         "--exclude-from",
         metavar="EXCLUDEFILE",
+        type=FilesystemPathSpec,
         action=ArgparseExcludeFileAction,
         help="read exclude patterns from EXCLUDEFILE, one per line",
     )
@@ -322,6 +324,7 @@ def define_exclude_and_patterns(add_option, *, tag_files=False, strip_components
     add_option(
         "--patterns-from",
         metavar="PATTERNFILE",
+        type=FilesystemPathSpec,
         action=ArgparsePatternFileAction,
         help="read include/exclude patterns from PATTERNFILE, one per line",
     )
diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index fade20ede2..b1a81abadf 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -7,6 +7,7 @@
 
 from ..constants import *  # NOQA
 from ..helpers import format_file_size, CompressionSpec
+from ..helpers import FilesystemDirSpec
 from ..helpers import json_print
 from ..helpers import msgpack
 from ..helpers import get_reset_ec
@@ -391,7 +392,9 @@ def build_parser_benchmarks(self, subparsers, common_parser, mid_common_parser):
             "crud", subparser, help="benchmarks Borg CRUD (create, extract, update, delete)."
         )
 
-        subparser.add_argument("path", metavar="PATH", help="path where to create benchmark input data")
+        subparser.add_argument(
+            "path", metavar="PATH", type=FilesystemDirSpec, help="path where to create benchmark input data"
+        )
         subparser.add_argument("--json-lines", action="store_true", help="Format output as JSON Lines.")
 
         bench_cpu_epilog = process_epilog(
diff --git a/src/borg/archiver/completion_cmd.py b/src/borg/archiver/completion_cmd.py
index e238cfa8bc..6a39c8f79d 100644
--- a/src/borg/archiver/completion_cmd.py
+++ b/src/borg/archiver/completion_cmd.py
@@ -2,8 +2,10 @@
 Shell completion support for Borg commands.
 
 This module implements the `borg completion` command, which generates shell completion
-scripts for bash and zsh. It uses the shtab library for basic completion generation
-and extends it with custom dynamic completions for Borg-specific argument types.
+scripts for bash, zsh, tcsh and fish. It uses the shtab library for basic completion
+generation and extends it with custom dynamic completions for Borg-specific argument
+types, via a per-shell preamble of completion functions that the generated script
+refers to.
 
 Dynamic Completions
 -------------------
@@ -13,7 +15,7 @@
 1. Archive names/IDs (archivename_validator):
    - Completes archive names by default (e.g., "my-backup-2024")
    - Completes archive IDs when prefixed with "aid:" (e.g., "aid:12345678")
-   - In zsh, shows archive metadata (name, timestamp, user@host) as descriptions
+   - In zsh and fish, shows archive metadata (name, timestamp, user@host) as descriptions
    - Respects --repo/-r flags to query the correct repository
 
 2. Sort keys (SortBySpec):
@@ -30,8 +32,10 @@
 5. Chunker parameters (ChunkerParams):
    - Suggests chunker param examples (default, fixed,4194304, buzhash,19,23,21,4095, etc.)
 
-6. Paths (PathSpec):
-   - Completes directories using standard shell directory completion
+6. Paths (PathSpec, FilesystemPathSpec, FilesystemDirSpec):
+   - Completes files and directories for arguments taking a filesystem path
+     (FilesystemPathSpec, e.g. PATH, EXCLUDEFILE, TARFILE), and directories only
+     for arguments taking a directory (FilesystemDirSpec, e.g. MOUNTPOINT)
 
 7. Help topics:
    - Completes help command topics and subcommand names
@@ -59,6 +63,8 @@
     SortBySpec,
     FilesCacheMode,
     PathSpec,
+    FilesystemPathSpec,
+    FilesystemDirSpec,
     ChunkerParams,
     CompressionSpec,
     tag_validator,
@@ -626,6 +632,163 @@
 }
 """
 
+# Global fish preamble providing the dynamic completion functions the generated
+# script refers to (the fish counterparts of the bash/zsh ones above).
+FISH_PREAMBLE_TMPL = r"""
+# derive repo context from the command line: --repo=V, --repo V, -r=V, -rV, or -r V
+function _borg_repo_args
+    set -l tokens (commandline -opc)
+    set -l n (count $tokens)
+    for i in (seq 2 $n)
+        set -l w $tokens[$i]
+        switch "$w"
+            case '--repo=*'
+                printf '%s\n' --repo (string replace -- '--repo=' '' $w)
+                return
+            case '-r=*'
+                printf '%s\n' -r (string replace -- '-r=' '' $w)
+                return
+            case '--repo' '-r'
+                if test $i -lt $n
+                    printf '%s\n' $w $tokens[(math $i + 1)]
+                end
+                return
+            case '-r*'
+                printf '%s\n' -r (string replace -r -- '^-r' '' $w)
+                return
+        end
+    end
+end
+
+# Complete archive names, or archive IDs when the current token starts with "aid:"
+function _borg_complete_archive
+    set -l cur (commandline -ct)
+    set -l repo_args (_borg_repo_args)
+    if string match -q -- 'aid:*' $cur
+        set -l prefix (string replace -- 'aid:' '' $cur)
+        if test -n "$prefix"; and not string match -qr -- '^[0-9a-fA-F]+$' $prefix
+            return
+        end
+        # ask borg for IDs with metadata; avoid prompts and suppress stderr
+        borg repo-list $repo_args --format '{id}{TAB}{archive}{TAB}{time}{TAB}{username}@{hostname}{NL}' \
+            2>/dev/null </dev/null | while read -l -d \t id archive atime userhost
+            test -z "$id"; and continue
+            # Print only the first 8 hex digits of the ID, with metadata as description.
+            printf 'aid:%s\t%s (%s %s)\n' (string sub -l 8 -- $id) $archive $atime $userhost
+        end
+    else
+        borg repo-list $repo_args --format '{archive}{NL}' 2>/dev/null </dev/null
+    end
+end
+
+# Complete tags from the repository
+function _borg_complete_tags
+    set -l repo_args (_borg_repo_args)
+    borg repo-list $repo_args --format '{tags}{NL}' 2>/dev/null </dev/null \
+        | string split ',' | string trim | string match -v '' | sort -u
+end
+
+# Complete compression spec options
+function _borg_complete_compression_spec
+    printf '%s\n' {COMP_SPEC_CHOICES}
+end
+
+# Complete chunker params options
+function _borg_complete_chunker_params
+    printf '%s\n' {CHUNKER_PARAMS_CHOICES}
+end
+
+# Complete relative time markers
+function _borg_complete_relative_time
+    printf '%s\n' {RELATIVE_TIME_CHOICES}
+end
+
+# Complete file size values
+function _borg_complete_file_size
+    printf '%s\n' {FILE_SIZE_CHOICES}
+end
+
+function _borg_help_topics
+    printf '%s\n' {HELP_CHOICES}
+end
+
+# Complete timestamp (file path or ISO timestamp)
+function _borg_complete_timestamp
+    set -l cur (commandline -ct)
+    if string match -qr -- '^(/|\.)' $cur
+        __fish_complete_path $cur
+    else
+        # Suggest current timestamp in ISO format
+        date +%Y-%m-%dT%H:%M:%S%z | string replace -r -- '(\d\d)$' ':$1'
+    end
+end
+
+# Split the current token into an --opt= prefix (if any), the already selected
+# comma-separated keys (with trailing comma) and set $_borg_csv_prefix_eq,
+# $_borg_csv_head and $_borg_csv_selected accordingly.
+function _borg_csv_scan
+    set -l cur (commandline -ct)
+    set -g _borg_csv_prefix_eq ''
+    set -l val $cur
+    if string match -q -- '*=*' $cur
+        set -g _borg_csv_prefix_eq (string replace -r -- '=.*$' '=' $cur)
+        set val (string replace -r -- '^[^=]*=' '' $cur)
+    end
+    set -g _borg_csv_head ''
+    if string match -q -- '*,*' $val
+        set -g _borg_csv_head (string replace -r -- '[^,]*$' '' $val)
+    end
+    set -g _borg_csv_selected (string split -- ',' $_borg_csv_head)
+end
+
+# Emit one candidate for a comma-separated list option, in both the plain and
+# the --opt=value form (fish matches whichever fits the current token).
+function _borg_csv_emit
+    printf '%s\n' "$_borg_csv_head$argv"
+    if test -n "$_borg_csv_prefix_eq"
+        printf '%s\n' "$_borg_csv_prefix_eq$_borg_csv_head$argv"
+    end
+end
+
+# Complete comma-separated sort keys for any option with type=SortBySpec.
+function _borg_complete_sortby
+    _borg_csv_scan
+    for k in {SORT_KEYS}
+        if contains -- $k $_borg_csv_selected
+            continue
+        end
+        _borg_csv_emit $k
+    end
+end
+
+# Complete comma-separated files cache mode tokens for options with type=FilesCacheMode.
+function _borg_complete_filescachemode
+    _borg_csv_scan
+    # If 'disabled' is already selected, there is nothing else to suggest.
+    if contains -- disabled $_borg_csv_selected
+        return
+    end
+    for k in {FCM_KEYS}
+        # skip duplicates
+        if contains -- $k $_borg_csv_selected
+            continue
+        end
+        # do not suggest 'disabled' if any other token is already selected
+        if test -n "$_borg_csv_head"; and test "$k" = disabled
+            continue
+        end
+        # ctime/mtime are mutually exclusive: don't suggest the other if one is present
+        if test "$k" = ctime; and contains -- mtime $_borg_csv_selected
+            continue
+        end
+        if test "$k" = mtime; and contains -- ctime $_borg_csv_selected
+            continue
+        end
+        _borg_csv_emit $k
+    end
+end
+"""
+
 
 def _attach_completion(parser: ArgumentParser, type_class, completion_dict: dict):
     """Tag all arguments with type `type_class` with completion choices from `completion_dict`."""
@@ -652,6 +815,12 @@ def _attach_help_completion(parser: ArgumentParser, completion_dict: dict):
             action.complete = completion_dict  # type: ignore[attr-defined]
 
 
+# File completion. shtab's fish variant calls __fish_complete_path without arguments,
+# which only completes paths relative to the cwd - pass the current token, so that
+# absolute paths (and paths in other directories) complete as well, see tqdm/shtab#245.
+FILE = {**shtab.FILE, "fish": "(__fish_complete_path (commandline -ct))"}
+
+
 class CompletionMixIn:
     def do_completion(self, args):
         """Output shell completion script for the given shell."""
@@ -659,33 +828,33 @@ def do_completion(self, args):
         # adds dynamic completion for archive IDs with the aid: prefix for all ARCHIVE
         # arguments (identified by archivename_validator). It reuses `borg repo-list`
         # to enumerate archives and does not introduce any new commands or caching.
-        parser = self.build_parser()
-        _attach_completion(
-            parser, archivename_validator, {"bash": "_borg_complete_archive", "zsh": "_borg_complete_archive"}
-        )
-        _attach_completion(parser, SortBySpec, {"bash": "_borg_complete_sortby", "zsh": "_borg_complete_sortby"})
-        _attach_completion(
-            parser, FilesCacheMode, {"bash": "_borg_complete_filescachemode", "zsh": "_borg_complete_filescachemode"}
-        )
-        _attach_completion(
-            parser,
-            CompressionSpec,
-            {"bash": "_borg_complete_compression_spec", "zsh": "_borg_complete_compression_spec"},
-        )
+
+        # The script shall always complete the "borg" command, no matter how borg was
+        # invoked while generating it (e.g. as "python -m borg", where prog would be
+        # "__main__.py" - the shells would then complete that instead of "borg").
+        prog, self.prog = self.prog, "borg"
+        try:
+            parser = self.build_parser()
+        finally:
+            self.prog = prog
+
+        def for_all_shells(fn_name):
+            # same-named completion function in the bash, zsh and fish preambles;
+            # fish needs a command substitution to call it
+            return {"bash": fn_name, "zsh": fn_name, "fish": f"({fn_name})"}
+
+        _attach_completion(parser, archivename_validator, for_all_shells("_borg_complete_archive"))
+        _attach_completion(parser, SortBySpec, for_all_shells("_borg_complete_sortby"))
+        _attach_completion(parser, FilesCacheMode, for_all_shells("_borg_complete_filescachemode"))
+        _attach_completion(parser, CompressionSpec, for_all_shells("_borg_complete_compression_spec"))
         _attach_completion(parser, PathSpec, shtab.DIRECTORY)
-        _attach_completion(
-            parser, ChunkerParams, {"bash": "_borg_complete_chunker_params", "zsh": "_borg_complete_chunker_params"}
-        )
-        _attach_completion(parser, tag_validator, {"bash": "_borg_complete_tags", "zsh": "_borg_complete_tags"})
-        _attach_completion(
-            parser,
-            relative_time_marker_validator,
-            {"bash": "_borg_complete_relative_time", "zsh": "_borg_complete_relative_time"},
-        )
-        _attach_completion(parser, timestamp, {"bash": "_borg_complete_timestamp", "zsh": "_borg_complete_timestamp"})
-        _attach_completion(
-            parser, parse_file_size, {"bash": "_borg_complete_file_size", "zsh": "_borg_complete_file_size"}
-        )
+        _attach_completion(parser, FilesystemPathSpec, FILE)
+        _attach_completion(parser, FilesystemDirSpec, shtab.DIRECTORY)
+        _attach_completion(parser, ChunkerParams, for_all_shells("_borg_complete_chunker_params"))
+        _attach_completion(parser, tag_validator, for_all_shells("_borg_complete_tags"))
+        _attach_completion(parser, relative_time_marker_validator, for_all_shells("_borg_complete_relative_time"))
+        _attach_completion(parser, timestamp, for_all_shells("_borg_complete_timestamp"))
+        _attach_completion(parser, parse_file_size, for_all_shells("_borg_complete_file_size"))
 
         # Collect all commands and help topics for "borg help" completion
         help_choices = list(self.helptext.keys())
@@ -693,8 +862,7 @@ def do_completion(self, args):
             if isinstance(action, ActionSubCommands):
                 help_choices.extend(action.choices.keys())
 
-        help_completion_fn = "_borg_help_topics"
-        _attach_help_completion(parser, {"bash": help_completion_fn, "zsh": help_completion_fn})
+        _attach_help_completion(parser, for_all_shells("_borg_help_topics"))
 
         # Build preambles using partial_format to avoid escaping braces etc.
         sort_keys = " ".join(AI_HUMAN_SORT_KEYS)
@@ -737,15 +905,10 @@ def do_completion(self, args):
             "FILE_SIZE_CHOICES": file_size_choices_str,
             "HELP_CHOICES": help_choices,
         }
-        bash_preamble = partial_format(BASH_PREAMBLE_TMPL, mapping)
-        zsh_preamble = partial_format(ZSH_PREAMBLE_TMPL, mapping)
-
-        if args.shell == "bash":
-            preambles = [bash_preamble]
-        elif args.shell == "zsh":
-            preambles = [zsh_preamble]
-        else:
-            preambles = []
+        preamble_templates = {"bash": BASH_PREAMBLE_TMPL, "zsh": ZSH_PREAMBLE_TMPL, "fish": FISH_PREAMBLE_TMPL}
+        template = preamble_templates.get(args.shell)
+        # Build the preamble using partial_format to avoid escaping braces etc.
+        preambles = [partial_format(template, mapping)] if template else []
         script = parser.get_completion_script(f"shtab-{args.shell}", preambles=preambles)
         print(script)
 
diff --git a/src/borg/archiver/debug_cmd.py b/src/borg/archiver/debug_cmd.py
index 3e0fb26548..40eb210176 100644
--- a/src/borg/archiver/debug_cmd.py
+++ b/src/borg/archiver/debug_cmd.py
@@ -4,6 +4,7 @@
 from ..archive import Archive
 from ..constants import *  # NOQA
 from ..helpers import msgpack
+from ..helpers import FilesystemPathSpec
 from ..helpers import sysinfo
 from ..helpers import bin_to_hex, hex_to_bin, prepare_dump_dict
 from ..helpers import dash_open
@@ -365,7 +366,7 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         )
         debug_parsers.add_subcommand("dump-archive", subparser, help="dump decoded archive metadata (debug)")
         subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name")
-        subparser.add_argument("path", metavar="PATH", type=str, help="file to dump data into")
+        subparser.add_argument("path", metavar="PATH", type=FilesystemPathSpec, help="file to dump data into")
 
         debug_dump_manifest_epilog = process_epilog(
             """
@@ -378,7 +379,7 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
             epilog=debug_dump_manifest_epilog,
         )
         debug_parsers.add_subcommand("dump-manifest", subparser, help="dump decoded repository metadata (debug)")
-        subparser.add_argument("path", metavar="PATH", type=str, help="file to dump data into")
+        subparser.add_argument("path", metavar="PATH", type=FilesystemPathSpec, help="file to dump data into")
 
         debug_dump_repo_objs_epilog = process_epilog(
             """
@@ -420,7 +421,7 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         )
         debug_parsers.add_subcommand("id-hash", subparser, help="compute id-hash for some file content (debug)")
         subparser.add_argument(
-            "path", metavar="PATH", type=str, help="content for which the id-hash shall get computed"
+            "path", metavar="PATH", type=FilesystemPathSpec, help="content for which the id-hash shall get computed"
         )
 
         # parse_obj
@@ -435,13 +436,22 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         debug_parsers.add_subcommand("parse-obj", subparser, help="parse borg object file into meta dict and data")
         subparser.add_argument("id", metavar="ID", type=str, help="hex object ID to get from the repo")
         subparser.add_argument(
-            "object_path", metavar="OBJECT_PATH", type=str, help="path of the object file to parse data from"
+            "object_path",
+            metavar="OBJECT_PATH",
+            type=FilesystemPathSpec,
+            help="path of the object file to parse data from",
         )
         subparser.add_argument(
-            "binary_path", metavar="BINARY_PATH", type=str, help="path of the file to write uncompressed data into"
+            "binary_path",
+            metavar="BINARY_PATH",
+            type=FilesystemPathSpec,
+            help="path of the file to write uncompressed data into",
         )
         subparser.add_argument(
-            "json_path", metavar="JSON_PATH", type=str, help="path of the json file to write metadata into"
+            "json_path",
+            metavar="JSON_PATH",
+            type=FilesystemPathSpec,
+            help="path of the json file to write metadata into",
         )
 
         # format_obj
@@ -456,10 +466,16 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         debug_parsers.add_subcommand("format-obj", subparser, help="format file and metadata into a Borg object file")
         subparser.add_argument("id", metavar="ID", type=str, help="hex object ID to get from the repo")
         subparser.add_argument(
-            "binary_path", metavar="BINARY_PATH", type=str, help="path of the file to convert into an object file"
+            "binary_path",
+            metavar="BINARY_PATH",
+            type=FilesystemPathSpec,
+            help="path of the file to convert into an object file",
         )
         subparser.add_argument(
-            "json_path", metavar="JSON_PATH", type=str, help="path of the json file to read metadata from"
+            "json_path",
+            metavar="JSON_PATH",
+            type=FilesystemPathSpec,
+            help="path of the json file to read metadata from",
         )
         subparser.add_argument(
             "-C",
@@ -474,7 +490,7 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         subparser.add_argument(
             "object_path",
             metavar="OBJECT_PATH",
-            type=str,
+            type=FilesystemPathSpec,
             help="path of the object file to write compressed encrypted data into",
         )
 
@@ -488,7 +504,7 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         )
         debug_parsers.add_subcommand("get-obj", subparser, help="get object from repository (debug)")
         subparser.add_argument("id", metavar="ID", type=str, help="hex object ID to get from the repo")
-        subparser.add_argument("path", metavar="PATH", type=str, help="file to write object data into")
+        subparser.add_argument("path", metavar="PATH", type=FilesystemPathSpec, help="file to write object data into")
 
         debug_put_obj_epilog = process_epilog(
             """
@@ -500,7 +516,9 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         )
         debug_parsers.add_subcommand("put-obj", subparser, help="put object to repository (debug)")
         subparser.add_argument("id", metavar="ID", type=str, help="hex object ID to put into the repo")
-        subparser.add_argument("path", metavar="PATH", type=str, help="file to read and create object from")
+        subparser.add_argument(
+            "path", metavar="PATH", type=FilesystemPathSpec, help="file to read and create object from"
+        )
 
         debug_delete_obj_epilog = process_epilog(
             """
@@ -528,5 +546,5 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser):
         debug_parsers.add_subcommand(
             "convert-profile", subparser, help="convert Borg profile to Python profile (debug)"
         )
-        subparser.add_argument("input", metavar="INPUT", type=str, help="Borg profile")
-        subparser.add_argument("output", metavar="OUTPUT", type=str, help="Output file")
+        subparser.add_argument("input", metavar="INPUT", type=FilesystemPathSpec, help="Borg profile")
+        subparser.add_argument("output", metavar="OUTPUT", type=FilesystemPathSpec, help="Output file")
diff --git a/src/borg/archiver/key_cmds.py b/src/borg/archiver/key_cmds.py
index 199d1845f8..40fed2f711 100644
--- a/src/borg/archiver/key_cmds.py
+++ b/src/borg/archiver/key_cmds.py
@@ -3,7 +3,7 @@
 from ..constants import *  # NOQA
 from ..crypto.key import KEY_LOCATIONS
 from ..crypto.keymanager import KeyManager
-from ..helpers import PathSpec, CommandError
+from ..helpers import FilesystemPathSpec, CommandError
 from ..helpers.argparsing import ArgumentParser
 from ..manifest import Manifest
 
@@ -189,7 +189,9 @@ def build_parser_keys(self, subparsers, common_parser, mid_common_parser):
             parents=[common_parser], description=self.do_key_export.__doc__, epilog=key_export_epilog
         )
         key_parsers.add_subcommand("export", subparser, help="export a borg key for backup")
-        subparser.add_argument("path", metavar="PATH", nargs="?", type=PathSpec, help="where to store the backup")
+        subparser.add_argument(
+            "path", metavar="PATH", nargs="?", type=FilesystemPathSpec, help="where to store the backup"
+        )
         export_select = subparser.add_mutually_exclusive_group()
         export_select.add_argument("--label", metavar="LABEL", dest="label", help="export the borg key with this label")
         export_select.add_argument(
@@ -231,7 +233,11 @@ def build_parser_keys(self, subparsers, common_parser, mid_common_parser):
         )
         key_parsers.add_subcommand("import", subparser, help="import the repository key from backup")
         subparser.add_argument(
-            "path", metavar="PATH", nargs="?", type=PathSpec, help="path to the backup ('-' to read from stdin)"
+            "path",
+            metavar="PATH",
+            nargs="?",
+            type=FilesystemPathSpec,
+            help="path to the backup ('-' to read from stdin)",
         )
         subparser.add_argument(
             "--paper",
diff --git a/src/borg/archiver/mount_cmds.py b/src/borg/archiver/mount_cmds.py
index e4d481741c..020bc9099b 100644
--- a/src/borg/archiver/mount_cmds.py
+++ b/src/borg/archiver/mount_cmds.py
@@ -3,7 +3,7 @@
 from ._common import with_repository, Highlander
 from ..constants import *  # NOQA
 from ..helpers import RTError
-from ..helpers import PathSpec
+from ..helpers import PathSpec, FilesystemDirSpec
 from ..helpers import umount
 from ..helpers.argparsing import ArgumentParser
 from ..manifest import Manifest
@@ -185,7 +185,7 @@ def build_parser_mount_umount(self, subparsers, common_parser, mid_common_parser
         subparser = ArgumentParser(parents=[common_parser], description=self.do_umount.__doc__, epilog=umount_epilog)
         subparsers.add_subcommand("umount", subparser, help="unmount a repository")
         subparser.add_argument(
-            "mountpoint", metavar="MOUNTPOINT", type=str, help="mountpoint of the filesystem to unmount"
+            "mountpoint", metavar="MOUNTPOINT", type=FilesystemDirSpec, help="mountpoint of the filesystem to unmount"
         )
 
     def build_parser_borgfs(self, parser):
@@ -199,7 +199,9 @@ def build_parser_borgfs(self, parser):
     def _define_borg_mount(self, parser):
         from ._common import define_exclusion_group, define_archive_filters_group
 
-        parser.add_argument("mountpoint", metavar="MOUNTPOINT", type=str, help="where to mount the filesystem")
+        parser.add_argument(
+            "mountpoint", metavar="MOUNTPOINT", type=FilesystemDirSpec, help="where to mount the filesystem"
+        )
         parser.add_argument(
             "-f", "--foreground", dest="foreground", action="store_true", help="stay in foreground, do not daemonize"
         )
diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py
index d085ffdbe9..f9caee421d 100644
--- a/src/borg/archiver/tar_cmds.py
+++ b/src/borg/archiver/tar_cmds.py
@@ -13,6 +13,7 @@
 from ..helpers import create_filter_process
 from ..helpers import ChunkIteratorFileWrapper
 from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams, CompressionSpec
+from ..helpers import FilesystemPathSpec
 from ..helpers import remove_surrogates
 from ..helpers import timestamp, archive_ts_now
 from ..helpers import basic_json_data, json_print
@@ -407,7 +408,9 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser):
             help="select tar format: BORG, PAX or GNU",
         )
         subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name")
-        subparser.add_argument("tarfile", metavar="FILE", help='output tar file. "-" to write to stdout instead.')
+        subparser.add_argument(
+            "tarfile", metavar="FILE", type=FilesystemPathSpec, help='output tar file. "-" to write to stdout instead.'
+        )
         subparser.add_argument(
             "paths", metavar="PATH", nargs="*", type=PathSpec, help="paths to extract; patterns are supported"
         )
@@ -536,4 +539,9 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser):
         )
 
         subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name")
-        subparser.add_argument("tarfile", metavar="TARFILE", help='input tar file. "-" to read from stdin instead.')
+        subparser.add_argument(
+            "tarfile",
+            metavar="TARFILE",
+            type=FilesystemPathSpec,
+            help='input tar file. "-" to read from stdin instead.',
+        )
diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py
index 886105faa9..703e2d839c 100644
--- a/src/borg/helpers/__init__.py
+++ b/src/borg/helpers/__init__.py
@@ -32,6 +32,7 @@
 from .parseformat import (
     PathSpec,
     FilesystemPathSpec,
+    FilesystemDirSpec,
     SortBySpec,
     CompressionSpec,
     ChunkerParams,
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index a4f2d539e2..a007e8a5b6 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -507,12 +507,21 @@ def __call__(self, text, overrides=None):
 
 
 def PathSpec(text):
+    """A path inside an archive."""
     if not text:
         raise ArgumentTypeError("Empty strings are not accepted as paths.")
     return text
 
 
 def FilesystemPathSpec(text):
+    """A path (file or directory) in the local filesystem."""
+    if not text:
+        raise ArgumentTypeError("Empty strings are not accepted as paths.")
+    return slashify(text)
+
+
+def FilesystemDirSpec(text):
+    """A directory in the local filesystem."""
     if not text:
         raise ArgumentTypeError("Empty strings are not accepted as paths.")
     return slashify(text)
diff --git a/src/borg/testsuite/archiver/completion_cmd_test.py b/src/borg/testsuite/archiver/completion_cmd_test.py
index ffa091199c..22e0b9570c 100644
--- a/src/borg/testsuite/archiver/completion_cmd_test.py
+++ b/src/borg/testsuite/archiver/completion_cmd_test.py
@@ -1,10 +1,12 @@
 import functools
 import os
 import subprocess
+import sys
 import tempfile
 
 import pytest
 
+import borg
 from . import cmd, generate_archiver_tests, RK_ENCRYPTION
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local")  # NOQA
@@ -21,7 +23,23 @@ def cmd_available(cmd):
 
 
 needs_bash = pytest.mark.skipif(not cmd_available("bash --version"), reason="Bash not available")
+
+
+@functools.lru_cache
+def bash_version():
+    """Major version of the bash on PATH (0 if unavailable)."""
+    try:
+        out = subprocess.run(["bash", "-c", "echo $BASH_VERSINFO"], capture_output=True, text=True, check=True)
+        return int(out.stdout.strip())
+    except (subprocess.SubprocessError, FileNotFoundError, ValueError):
+        return 0
+
+
+# the completion script shtab generates uses mapfile, which needs bash >= 4
+# (macOS still ships bash 3.2)
+needs_bash4 = pytest.mark.skipif(bash_version() < 4, reason="Bash >= 4 not available")
 needs_zsh = pytest.mark.skipif(not cmd_available("zsh --version"), reason="Zsh not available")
+needs_fish = pytest.mark.skipif(not cmd_available("fish --version"), reason="Fish not available")
 
 
 def _run_bash_completion_fn(completion_script, setup_code):
@@ -38,6 +56,42 @@ def _run_bash_completion_fn(completion_script, setup_code):
     return result
 
 
+def _fish_quote(text):
+    """Escape text for inclusion in a single-quoted fish string."""
+    return text.replace("\\", "\\\\").replace("'", "\\'")
+
+
+def _fish_complete_candidates(completion_script, cmdline, path_prepend=None):
+    """Source the completion script in fish, return the completion candidates for cmdline."""
+    code = completion_script + f"\ncomplete -C'{_fish_quote(cmdline)}'\n"
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".fish", delete=False) as f:
+        f.write(code)
+        script_path = f.name
+    env = dict(os.environ)
+    if path_prepend:
+        env["PATH"] = path_prepend + os.pathsep + env.get("PATH", "")
+    try:
+        result = subprocess.run(["fish", script_path], capture_output=True, text=True, timeout=120, env=env)
+    finally:
+        os.unlink(script_path)
+    assert result.returncode == 0, f"fish failed: {result.stderr}"
+    # each output line is "candidate<TAB>description" (or just "candidate")
+    return [line.split("\t")[0] for line in result.stdout.splitlines() if line.strip()]
+
+
+def _borg_shim_dir(tmp_path):
+    """Create a dir with a borg script that runs the borg under test, for use inside fish."""
+    src_dir = os.path.dirname(os.path.dirname(os.path.abspath(borg.__file__)))
+    shim_dir = tmp_path / "borg-shim"
+    shim_dir.mkdir()
+    shim = shim_dir / "borg"
+    shim.write_text(
+        f'#!/bin/sh\nexport PYTHONPATH="{src_dir}${{PYTHONPATH:+:$PYTHONPATH}}"\nexec "{sys.executable}" -m borg "$@"\n'
+    )
+    shim.chmod(0o755)
+    return str(shim_dir)
+
+
 # -- output sanity checks -----------------------------------------------------
 
 
@@ -57,6 +111,14 @@ def test_zsh_completion_nontrivial(archivers, request):
     assert output.count("\n") > 100, f"Zsh completion suspiciously few lines: {output.count(chr(10))}"
 
 
+def test_fish_completion_nontrivial(archivers, request):
+    """Verify the generated Fish completion is non-trivially sized."""
+    archiver = request.getfixturevalue(archivers)
+    output = cmd(archiver, "completion", "fish")
+    assert len(output) > 5000, f"Fish completion suspiciously small: {len(output)} chars"
+    assert output.count("\n") > 100, f"Fish completion suspiciously few lines: {output.count(chr(10))}"
+
+
 # -- syntax validation --------------------------------------------------------
 
 
@@ -90,6 +152,15 @@ def test_zsh_completion_syntax(archivers, request):
     assert result.returncode == 0, f"Generated Zsh completion has syntax errors: {result.stderr.decode()}"
 
 
+@needs_fish
+def test_fish_completion_syntax(archivers, request):
+    """Verify the generated Fish completion script has valid syntax."""
+    archiver = request.getfixturevalue(archivers)
+    output = cmd(archiver, "completion", "fish")
+    result = _check_shell_syntax(output, "fish", ".fish")
+    assert result.returncode == 0, f"Generated Fish completion has syntax errors: {result.stderr.decode()}"
+
+
 # -- borg-specific preamble function behavior (bash) --------------------------
 
 
@@ -182,3 +253,137 @@ def test_bash_archive_aid_completion(archivers, request):
     assert len(lines) >= 1, "Expected at least one archive ID completion"
     for line in lines:
         assert line.startswith("aid:"), f"Expected aid: prefix, got: {line}"
+
+
+# -- borg-specific completion behavior (fish) ---------------------------------
+
+
+@needs_fish
+def test_fish_subcommand_completion(archivers, request):
+    """Command names should be completed at the top level and inside command groups."""
+    archiver = request.getfixturevalue(archivers)
+    script = cmd(archiver, "completion", "fish")
+
+    candidates = _fish_complete_candidates(script, "borg ")
+    for name in ("create", "extract", "repo-list", "key", "debug"):
+        assert name in candidates, f"expected command {name} in: {candidates}"
+
+    candidates = _fish_complete_candidates(script, "borg key ")
+    for name in ("export", "import", "change-passphrase"):
+        assert name in candidates, f"expected key subcommand {name} in: {candidates}"
+
+
+@needs_fish
+def test_fish_option_completion(archivers, request):
+    """Option names and static option values should be completed."""
+    archiver = request.getfixturevalue(archivers)
+    script = cmd(archiver, "completion", "fish")
+
+    candidates = _fish_complete_candidates(script, "borg create --compres")
+    assert "--compression" in candidates, f"expected --compression in: {candidates}"
+
+    candidates = _fish_complete_candidates(script, "borg create --compression ")
+    assert "lz4" in candidates, f"expected lz4 in: {candidates}"
+    assert "zstd,3" in candidates, f"expected zstd,3 in: {candidates}"
+
+
+@needs_fish
+def test_fish_sortby_dedup(archivers, request):
+    """_borg_complete_sortby should not re-offer already-selected sort keys."""
+    archiver = request.getfixturevalue(archivers)
+    script = cmd(archiver, "completion", "fish")
+
+    candidates = _fish_complete_candidates(script, "borg repo-list --sort-by timestamp,")
+    assert "timestamp,archive" in candidates, f"expected timestamp,archive in: {candidates}"
+    assert "timestamp,timestamp" not in candidates, f"timestamp was re-offered: {candidates}"
+
+
+@needs_fish
+def test_fish_filescachemode_exclusivity(archivers, request):
+    """_borg_complete_filescachemode should enforce ctime/mtime and disabled mutual exclusion."""
+    archiver = request.getfixturevalue(archivers)
+    script = cmd(archiver, "completion", "fish")
+
+    candidates = _fish_complete_candidates(script, "borg create --files-cache ctime,")
+    assert "ctime,size" in candidates, f"expected ctime,size in: {candidates}"
+    assert "ctime,mtime" not in candidates, f"mtime offered after ctime: {candidates}"
+    assert "ctime,disabled" not in candidates, f"disabled offered after ctime: {candidates}"
+
+    candidates = _fish_complete_candidates(script, "borg create --files-cache disabled,")
+    assert candidates == [], f"completions offered after disabled: {candidates}"
+
+
+@needs_fish
+def test_fish_path_completion(archivers, request, tmp_path):
+    """Arguments taking a filesystem path complete files/directories."""
+    archiver = request.getfixturevalue(archivers)
+    (tmp_path / "somefile.txt").touch()
+    (tmp_path / "somedir").mkdir()
+    script = cmd(archiver, "completion", "fish")
+    prefix = str(tmp_path / "some")
+
+    # positional PATH of "borg create" ...
+    candidates = _fish_complete_candidates(script, f"borg create archivename {prefix}")
+    assert any(c.endswith("somefile.txt") for c in candidates), f"file missing: {candidates}"
+    # ... and the value of an option taking a file
+    candidates = _fish_complete_candidates(script, f"borg create --exclude-from {prefix}")
+    assert any(c.endswith("somefile.txt") for c in candidates), f"file missing: {candidates}"
+    # arguments taking a directory only offer directories
+    candidates = _fish_complete_candidates(script, f"borg mount --repo /repo archivename {prefix}")
+    assert any(c.rstrip("/").endswith("somedir") for c in candidates), f"dir missing: {candidates}"
+    assert not any(c.endswith("somefile.txt") for c in candidates), f"file offered for MOUNTPOINT: {candidates}"
+
+
+@needs_bash4
+def test_bash_path_completion(archivers, request, tmp_path):
+    """Arguments taking a filesystem path complete files in bash, too."""
+    archiver = request.getfixturevalue(archivers)
+    (tmp_path / "somefile.txt").touch()
+    script = cmd(archiver, "completion", "bash")
+    prefix = str(tmp_path / "some")
+
+    result = _run_bash_completion_fn(
+        script,
+        f'COMP_WORDS=(borg create --exclude-from "{prefix}")\n'
+        f"COMP_CWORD=3\n"
+        f"_shtab_borg\n"
+        f'printf "%s\\n" "${{COMPREPLY[@]}}"\n',
+    )
+    assert result.returncode == 0, f"stderr: {result.stderr}"
+    assert "somefile.txt" in result.stdout, f"file missing: {result.stdout}"
+
+
+@needs_fish
+def test_fish_archive_name_completion(archivers, request, tmp_path):
+    """Archive names should be completed from a real repo."""
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "mybackup-2024", archiver.input_path)
+    cmd(archiver, "create", "mybackup-2025", archiver.input_path)
+
+    script = cmd(archiver, "completion", "fish")
+    repo = archiver.repository_path
+
+    candidates = _fish_complete_candidates(
+        script, f"borg delete --repo {repo} mybackup", path_prepend=_borg_shim_dir(tmp_path)
+    )
+    assert "mybackup-2024" in candidates, f"archive name missing: {candidates}"
+    assert "mybackup-2025" in candidates, f"archive name missing: {candidates}"
+
+
+@needs_fish
+def test_fish_archive_aid_completion(archivers, request, tmp_path):
+    """aid: prefixed archive IDs should be completed from a real repo."""
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "testarchive", archiver.input_path)
+
+    script = cmd(archiver, "completion", "fish")
+    repo = archiver.repository_path
+
+    candidates = _fish_complete_candidates(
+        script, f"borg info --repo {repo} aid:", path_prepend=_borg_shim_dir(tmp_path)
+    )
+    assert len(candidates) >= 1, "Expected at least one archive ID completion"
+    for candidate in candidates:
+        assert candidate.startswith("aid:"), f"Expected aid: prefix, got: {candidate}"
diff --git a/src/borg/testsuite/shell_completions_test.py b/src/borg/testsuite/shell_completions_test.py
deleted file mode 100644
index 4fa26faa5a..0000000000
--- a/src/borg/testsuite/shell_completions_test.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import subprocess
-from pathlib import Path
-
-import pytest
-
-SHELL_COMPLETIONS_DIR = Path(__file__).parent / ".." / ".." / ".." / "scripts" / "shell_completions"
-
-
-def test_fish_completion_is_valid():
-    """Test that the Fish completion file is valid Fish syntax."""
-    fish_completion_file = SHELL_COMPLETIONS_DIR / "fish" / "borg.fish"
-    assert fish_completion_file.is_file()
-
-    # Check if Fish is available
-    try:
-        subprocess.run(["fish", "--version"], capture_output=True, check=True)
-    except (subprocess.SubprocessError, FileNotFoundError):
-        pytest.skip("Fish not available")
-
-    # Test whether the Fish completion file can be sourced without errors
-    result = subprocess.run(["fish", "-c", f"source {str(fish_completion_file)}"], capture_output=True)
-    assert result.returncode == 0, f"Fish completion file has syntax errors: {result.stderr.decode()}"

From bfb0f5cc5f9b2e95694057338b38110830740d92 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 18 Mar 2026 12:56:49 +0100
Subject: [PATCH 019/144] add tcsh completion support

`borg completion tcsh` now generates a usable completion script:

- add a tcsh preamble with the dynamic completion helpers (as aliases, since tcsh has no
  functions) and wire the tcsh patterns up for sort keys, files-cache mode, compression specs,
  chunker params, relative times, timestamps, file sizes and help topics.
- complete archive names, archive IDs (when the token starts with "aid:") and tags. tcsh can
  neither define functions nor use backquotes there (the completion rule calling the helper is
  backquoted already), so both helpers run one POSIX sh script that parses $COMMAND_LINE for
  --repo/-r and queries `borg repo-list`. tcsh has no completion descriptions, so unlike zsh
  and fish these are plain candidate lists.

The generator fixes this needed are all upstream in shtab now (tqdm/shtab#213, released in
1.9.3, which we already require): positional completion under subcommands at any depth, no
out-of-range `$cmd` indexing, custom `.complete` patterns in multi-requirement rules, `--opt=`
completion, and rule deduplication.

One upstream fix is merged but not yet released (tqdm/shtab#241): completion patterns (`f`,
`d`, ...) for a positional of a subcommand end up inside a `p@N@` rule, where tcsh runs the
clauses as commands and only uses their output, so they do nothing - e.g. `borg umount <TAB>`
would not complete a mountpoint. `_tcsh_anchor_positional_patterns` rewrites those into `n/`
rules keyed off the preceding (sub)command word, producing exactly what a shtab with #241
generates. It is a no-op with such a shtab and can then be removed.

Note that tcsh matches completions for positional arguments by word position, so options
before an archive name shift it out of place and it is not completed - the `borg completion`
epilog points at BORG_REPO for this.
---
 src/borg/archiver/completion_cmd.py           | 120 +++++++++++++++++-
 .../testsuite/archiver/completion_cmd_test.py |  56 ++++++++
 2 files changed, 172 insertions(+), 4 deletions(-)

diff --git a/src/borg/archiver/completion_cmd.py b/src/borg/archiver/completion_cmd.py
index 6a39c8f79d..c50287735d 100644
--- a/src/borg/archiver/completion_cmd.py
+++ b/src/borg/archiver/completion_cmd.py
@@ -16,6 +16,7 @@
    - Completes archive names by default (e.g., "my-backup-2024")
    - Completes archive IDs when prefixed with "aid:" (e.g., "aid:12345678")
    - In zsh and fish, shows archive metadata (name, timestamp, user@host) as descriptions
+     (tcsh has no completion descriptions)
    - Respects --repo/-r flags to query the correct repository
 
 2. Sort keys (SortBySpec):
@@ -54,6 +55,8 @@
    - Suggests common file size values (500M, 1G, 10G, 100G, 1T, etc.)
 """
 
+import re
+
 import shtab
 
 from ._common import process_epilog
@@ -790,6 +793,103 @@
 """
 
 
+TCSH_PREAMBLE_TMPL = r"""
+# Dynamic completion helpers for tcsh
+
+alias _borg_complete_timestamp 'date +"%Y-%m-%dT%H:%M:%S"'
+
+
+alias _borg_complete_sortby "echo {SORT_KEYS}"
+alias _borg_complete_filescachemode "echo {FCM_KEYS}"
+alias _borg_help_topics "echo {HELP_CHOICES}"
+alias _borg_complete_compression_spec "echo {COMP_SPEC_CHOICES}"
+alias _borg_complete_chunker_params "echo {CHUNKER_PARAMS_CHOICES}"
+alias _borg_complete_relative_time "echo {RELATIVE_TIME_CHOICES}"
+alias _borg_complete_file_size "echo {FILE_SIZE_CHOICES}"
+
+# Complete archive names (archive IDs when the current token starts with "aid:") and tags.
+# These need the command line (for --repo/-r) and some logic, which tcsh cannot do itself:
+# it has no functions, and an alias cannot use backquotes here because the completion rule
+# calling the alias is backquoted already. So the work is done by a POSIX sh script, kept in
+# a variable (a single-quoted csh string, hence no single quotes in it) and run via "sh -c".
+set _borg_sh_complete = '{SH_COMPLETE}'
+
+alias _borg_complete_archive 'sh -c "$_borg_sh_complete" borg-completion archive "$COMMAND_LINE"'
+alias _borg_complete_tags 'sh -c "$_borg_sh_complete" borg-completion tags "$COMMAND_LINE"'
+"""
+
+# the sh script the tcsh preamble runs, as one line (`sh -c <script> borg-completion <mode> <line>`).
+# It must not contain single quotes (it goes into a single-quoted csh string) nor backquotes (the
+# completion rule invoking it is backquoted already).
+TCSH_SH_COMPLETE = (
+    "mode=$1; line=$2; "
+    # derive repo context from the command line: --repo=V, --repo V, -r=V, -rV, or -r V
+    "repo=; prev=; "
+    "for w in $line; do "
+    "case $w in --repo=*) repo=${w#--repo=};; -r=*) repo=${w#-r=};; -r?*) repo=${w#-r};; esac; "
+    "case $prev in --repo|-r) repo=$w;; esac; "
+    "prev=$w; "
+    "done; "
+    'set --; if [ -n "$repo" ]; then set -- --repo "$repo"; fi; '
+    # the token being completed: the last one, empty if the line ends with a space
+    "cur=${line##* }; "
+    # avoid prompts and suppress errors, the output is used as completion candidates
+    "if [ $mode = tags ]; then "
+    'borg repo-list "$@" --format "{tags}{NL}" 2>/dev/null </dev/null'
+    ' | tr , "\\n" | sed "s/^ *//;s/ *$//" | grep . | sort -u; '
+    'elif [ "${cur#aid:}" != "$cur" ]; then '
+    # print only the first 8 hex digits of the ID, like the other shells do
+    'borg repo-list "$@" --format "aid:{id}{NL}" 2>/dev/null </dev/null | cut -c1-12; '
+    "else "
+    'borg repo-list "$@" --format "{archive}{NL}" 2>/dev/null </dev/null; '
+    "fi"
+)
+
+
+# in a `p@N@`...`@` rule, one `if (...) <action>` clause, e.g.
+# `if ( $#cmd >= 3 && ("$cmd[2]" == "key") && ("$cmd[3]" == "export") ) f`
+TCSH_CLAUSE_RE = re.compile(r"if \( \$#cmd >= \d+ && (?P<checks>.*?) \) (?P<action>.*)")
+TCSH_CHECK_RE = re.compile(r'\("\$cmd\[\d+\]" == "(?P<word>[^"]+)"\)')
+# a `p@N@` rule as a whole
+TCSH_RULE_RE = re.compile(r"^(?P<indent>\s*)'p@(?P<idx>\d+)@`(?P<setup>set cmd=[^;]+; )(?P<clauses>.*)`@' \\$")
+
+
+def _tcsh_anchor_positional_patterns(script):
+    """
+    Complete files/dirs for positionals of a subcommand, e.g. `borg umount <MOUNTPOINT>`.
+
+    shtab puts such completions into the `p@N@` rule of that positional, as a bare completion
+    pattern (`f`, `d`, ...) in an `if (...) <action>` clause. But tcsh runs these clauses as
+    commands and only uses their *output*, so a pattern there does nothing. tcsh can only apply
+    a pattern via a rule of its own, keyed off the preceding word - which works whenever the
+    positional directly follows its (sub)command.
+
+    TODO: remove this once we require a shtab release that includes tqdm/shtab#241 - with such
+    a shtab, no clause has a bare pattern as its action and this is a no-op.
+    """
+    lines = []
+    for line in script.splitlines():
+        rule = TCSH_RULE_RE.match(line)
+        if not rule:
+            lines.append(line)
+            continue
+        keep = []
+        for clause in rule["clauses"].split("; "):
+            match = TCSH_CLAUSE_RE.fullmatch(clause)
+            action = match["action"] if match else None
+            if action is None or action.startswith(("echo ", "eval ")):
+                keep.append(clause)  # not a pattern, tcsh can run this
+                continue
+            words = TCSH_CHECK_RE.findall(match["checks"])
+            # the positional is at index `idx`, the (sub)command words are at 2..len(words) + 1
+            if words and int(rule["idx"]) == len(words) + 1:
+                lines.append(f"{rule['indent']}'n/{words[-1]}/{action}/' \\")
+            # else: the pattern is for a later positional, tcsh cannot express that - drop it
+        if keep:
+            lines.append(f"{rule['indent']}'p@{rule['idx']}@`{rule['setup']}{'; '.join(keep)}`@' \\")
+    return "\n".join(lines)
+
+
 def _attach_completion(parser: ArgumentParser, type_class, completion_dict: dict):
     """Tag all arguments with type `type_class` with completion choices from `completion_dict`."""
 
@@ -839,9 +939,9 @@ def do_completion(self, args):
             self.prog = prog
 
         def for_all_shells(fn_name):
-            # same-named completion function in the bash, zsh and fish preambles;
-            # fish needs a command substitution to call it
-            return {"bash": fn_name, "zsh": fn_name, "fish": f"({fn_name})"}
+            # same-named completion helper in the bash, zsh, tcsh and fish preambles;
+            # fish needs a command substitution to call it, tcsh backquotes
+            return {"bash": fn_name, "zsh": fn_name, "tcsh": f"`{fn_name}`", "fish": f"({fn_name})"}
 
         _attach_completion(parser, archivename_validator, for_all_shells("_borg_complete_archive"))
         _attach_completion(parser, SortBySpec, for_all_shells("_borg_complete_sortby"))
@@ -904,12 +1004,20 @@ def for_all_shells(fn_name):
             "RELATIVE_TIME_CHOICES": relative_time_choices_str,
             "FILE_SIZE_CHOICES": file_size_choices_str,
             "HELP_CHOICES": help_choices,
+            "SH_COMPLETE": TCSH_SH_COMPLETE,
+        }
+        preamble_templates = {
+            "bash": BASH_PREAMBLE_TMPL,
+            "zsh": ZSH_PREAMBLE_TMPL,
+            "tcsh": TCSH_PREAMBLE_TMPL,
+            "fish": FISH_PREAMBLE_TMPL,
         }
-        preamble_templates = {"bash": BASH_PREAMBLE_TMPL, "zsh": ZSH_PREAMBLE_TMPL, "fish": FISH_PREAMBLE_TMPL}
         template = preamble_templates.get(args.shell)
         # Build the preamble using partial_format to avoid escaping braces etc.
         preambles = [partial_format(template, mapping)] if template else []
         script = parser.get_completion_script(f"shtab-{args.shell}", preambles=preambles)
+        if args.shell == "tcsh":
+            script = _tcsh_anchor_positional_patterns(script)
         print(script)
 
     def build_parser_completion(self, subparsers, common_parser, mid_common_parser):
@@ -923,6 +1031,10 @@ def build_parser_completion(self, subparsers, common_parser, mid_common_parser):
         completion script will call borg to query the repository. This will work best
         if that call can be made without prompting for user input, so you may want to
         set BORG_REPO and BORG_PASSPHRASE environment variables.
+
+        In tcsh, completions for positional arguments are matched by word position, so
+        e.g. an archive name is only completed if no options precede it - one more
+        reason to use BORG_REPO there rather than --repo.
         """
         )
 
diff --git a/src/borg/testsuite/archiver/completion_cmd_test.py b/src/borg/testsuite/archiver/completion_cmd_test.py
index 22e0b9570c..2260f8fb50 100644
--- a/src/borg/testsuite/archiver/completion_cmd_test.py
+++ b/src/borg/testsuite/archiver/completion_cmd_test.py
@@ -1,5 +1,6 @@
 import functools
 import os
+import re
 import subprocess
 import sys
 import tempfile
@@ -40,6 +41,7 @@ def bash_version():
 needs_bash4 = pytest.mark.skipif(bash_version() < 4, reason="Bash >= 4 not available")
 needs_zsh = pytest.mark.skipif(not cmd_available("zsh --version"), reason="Zsh not available")
 needs_fish = pytest.mark.skipif(not cmd_available("fish --version"), reason="Fish not available")
+needs_tcsh = pytest.mark.skipif(not cmd_available("tcsh --version"), reason="Tcsh not available")
 
 
 def _run_bash_completion_fn(completion_script, setup_code):
@@ -119,6 +121,42 @@ def test_fish_completion_nontrivial(archivers, request):
     assert output.count("\n") > 100, f"Fish completion suspiciously few lines: {output.count(chr(10))}"
 
 
+def test_tcsh_completion_nontrivial(archivers, request):
+    """Verify the generated Tcsh completion is non-trivially sized."""
+    archiver = request.getfixturevalue(archivers)
+    output = cmd(archiver, "completion", "tcsh")
+    assert len(output) > 1000, f"Tcsh completion suspiciously small: {len(output)} chars"
+    assert output.count("\n") > 20, f"Tcsh completion suspiciously few lines: {output.count(chr(10))}"
+
+
+def test_tcsh_completion_dynamic_helpers(archivers, request):
+    """Verify the tcsh script has the dynamic archive/tag helpers and uses them."""
+    archiver = request.getfixturevalue(archivers)
+    output = cmd(archiver, "completion", "tcsh")
+    assert "alias _borg_complete_archive " in output, "no archive completion helper"
+    assert "alias _borg_complete_tags " in output, "no tag completion helper"
+    # the helper scripts are single-quoted csh strings run by sh, see the tcsh preamble
+    helper = output.split("set _borg_sh_complete = '", 1)[1].split("'", 1)[0]
+    assert "aid:{id}{NL}" in helper and "{archive}{NL}" in helper and "{tags}{NL}" in helper
+    assert "`" not in helper, "backquotes in the helper would nest inside the completion rules"
+    assert "eval _borg_complete_archive" in output, "archive completion not used for ARCHIVE"
+    assert "'n/--tags/`_borg_complete_tags`/'" in output, "tag completion not used for --tags"
+
+
+def test_tcsh_completion_positional_patterns(archivers, request):
+    """Verify positionals of a subcommand get a rule tcsh can apply, e.g. the umount mountpoint."""
+    archiver = request.getfixturevalue(archivers)
+    output = cmd(archiver, "completion", "tcsh")
+    assert "'n/umount/d/'" in output, "no directory completion for the umount mountpoint"
+    assert "'n/export/f/'" in output, "no file completion for the key export path"
+    # such a pattern is useless inside a `p@N@` rule, tcsh runs those clauses as commands
+    for rule in re.findall(r"'p@\d+@`(.*?)`@'", output):
+        _setup, _, clauses = rule.partition("; ")  # drop the `set cmd=(...)` part
+        for clause in clauses.split("; "):
+            action = clause.rpartition(") ")[2]
+            assert action.startswith(("echo ", "eval ")), f"bare completion pattern in rule: {rule}"
+
+
 # -- syntax validation --------------------------------------------------------
 
 
@@ -161,6 +199,24 @@ def test_fish_completion_syntax(archivers, request):
     assert result.returncode == 0, f"Generated Fish completion has syntax errors: {result.stderr.decode()}"
 
 
+@needs_tcsh
+def test_tcsh_completion_syntax(archivers, request):
+    """Verify the generated Tcsh completion script has valid syntax."""
+    archiver = request.getfixturevalue(archivers)
+    output = cmd(archiver, "completion", "tcsh")
+    # tcsh doesn't have -n for syntax check like bash/zsh, but we can try to source it
+    # and see if it fails. 'tcsh -f -c "source path"'
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".tcsh", delete=False) as f:
+        f.write(output)
+        script_path = f.name
+    try:
+        # -f: fast start (don't resource .tcshrc)
+        result = subprocess.run(["tcsh", "-f", "-c", f"source {script_path}"], capture_output=True)
+    finally:
+        os.unlink(script_path)
+    assert result.returncode == 0, f"Generated Tcsh completion has errors: {result.stderr.decode()}"
+
+
 # -- borg-specific preamble function behavior (bash) --------------------------
 
 

From 710c272ac5100c536138f9203eab2fe6e9f5221f Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 5 Aug 2026 13:08:01 +0200
Subject: [PATCH 020/144] lrucache: assigning to a key that is present replaces
 its value

LRUCache refused that with an assertion, so callers had to write

    if key in cache:
        cache.replace(key, value)
    else:
        cache[key] = value

and a caller that just assigned crashed - which is a nasty failure mode for a
cache: it is a correctness-neutral operation that suddenly is not. It bites
especially when several threads use one cache (see the FUSE data cache), where
two threads can miss on the same key and both insert the value they fetched.

Assignment now replaces the value, disposing the old one (it left the cache,
just like a deleted one would) unless it is the identical object, and counts as
a use, so the entry moves to the most-recently-used end.

replace() stays for the one case that really needs it: updating an entry while
keeping its old value alive - the legacy repository's fd cache re-timestamps its
entries that way and must not have the open file closed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/borg/helpers/lrucache.py                | 29 +++++++++++++++------
 src/borg/testsuite/helpers/lrucache_test.py | 27 +++++++++++++++++++
 2 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/src/borg/helpers/lrucache.py b/src/borg/helpers/lrucache.py
index a3bf1edb62..d06522bd84 100644
--- a/src/borg/helpers/lrucache.py
+++ b/src/borg/helpers/lrucache.py
@@ -9,8 +9,10 @@
 class LRUCache(MutableMapping[K, V]):
     """
     Mapping which maintains a maximum size by removing the least recently used value.
-    Items are passed to dispose before being removed and setting an item which is
-    already in the cache has to be done using the replace method.
+
+    A value that leaves the cache - evicted, deleted, or replaced by another value under
+    the same key - is passed to *dispose* first, so that a cache of e.g. open files can
+    close them. Use replace() to update an entry while keeping its old value alive.
     """
 
     _cache: OrderedDict[K, V]
@@ -25,12 +27,17 @@ def __init__(self, capacity: int, dispose: Callable[[V], None] = lambda _: None)
         self._dispose = dispose
 
     def __setitem__(self, key: K, value: V) -> None:
-        assert (
-            key not in self._cache
-        ), "Unexpected attempt to replace a cached item without first deleting the old item."
+        try:
+            previous = self._cache.pop(key)
+        except KeyError:
+            pass
+        else:
+            # the old value is no longer in the cache, so it is disposed like a deleted one.
+            if previous is not value:
+                self._dispose(previous)
         while len(self._cache) >= self._capacity:
             self._dispose(self._cache.popitem(last=False)[1])
-        self._cache[key] = value  # add new entry at the end
+        self._cache[key] = value  # add the new (or refreshed) entry at the end
 
     def __getitem__(self, key: K) -> V:
         self._cache.move_to_end(key)  # raise KeyError if not found
@@ -46,8 +53,14 @@ def __len__(self) -> int:
         return len(self._cache)
 
     def replace(self, key: K, value: V) -> None:
-        """Replace an item that is already present, not disposing it in the process."""
-        # this method complements __setitem__ which should be used for the normal use case.
+        """Replace the value of an entry that is present, without disposing the old value.
+
+        This is for the rare case where the old value must stay alive, e.g. because the new
+        value still refers to it (the legacy repository's fd cache re-times-stamps its
+        entries this way, keeping the open file object). It also keeps the entry where it
+        is in the LRU order, i.e. it does not count as a use. Everything else should just
+        assign to the cache.
+        """
         assert key in self._cache, "Unexpected attempt to update a non-existing item."
         self._cache[key] = value
 
diff --git a/src/borg/testsuite/helpers/lrucache_test.py b/src/borg/testsuite/helpers/lrucache_test.py
index 8897702f26..e152a3189f 100644
--- a/src/borg/testsuite/helpers/lrucache_test.py
+++ b/src/borg/testsuite/helpers/lrucache_test.py
@@ -35,6 +35,33 @@ def test_lrucache(self):
         c.clear()
         assert c.items() == set()
 
+    def test_assign_to_existing_key(self):
+        # assigning to a key that is present replaces the value and counts as a use,
+        # so the entry becomes the most recently used one.
+        c = LRUCache(2)
+        c["a"] = 1
+        c["b"] = 2
+        c["a"] = 3  # replaces, and refreshes "a"
+        assert len(c) == 2
+        assert c["a"] == 3
+        c["c"] = 4  # evicts the least recently used entry, which is "b" now
+        assert set(c.keys()) == {"a", "c"}
+
+    def test_dispose_on_replacement(self):
+        disposed = []
+        c = LRUCache(2, dispose=disposed.append)
+        c["a"] = "first"
+        c["a"] = "second"  # the replaced value leaves the cache, so it is disposed
+        assert disposed == ["first"]
+        assert c["a"] == "second"
+        value = "same object"
+        c["b"] = value
+        c["b"] = value  # assigning the identical object does not dispose it
+        assert disposed == ["first"]
+        c.replace("b", "replacement")  # replace() never disposes, see there
+        assert disposed == ["first"]
+        assert c["b"] == "replacement"
+
     def test_dispose(self):
         c = LRUCache(2, dispose=lambda f: f.close())
         f1 = TemporaryFile()

From fce88eeff4283aaf8265329ce5052cd92b6fbe93 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 5 Aug 2026 13:26:03 +0200
Subject: [PATCH 021/144] lrucache: make it thread-safe

Several threads share one LRUCache in borg: mfusepy serves the FUSE requests of
one mount from a thread pool, and "borg webdav" from one thread per connection.
Until now every caller had to serialize the access itself - a rule that is easy
to miss, and missing it does not fail loudly at the cache, it fails somewhere
far away (mfusepy turns any exception from a FUSE operation into EINVAL, so a
mount just reports "Invalid argument" for a healthy file).

Getting, setting, deleting, popping, replacing and clearing now each hold a lock,
so they are atomic. A *sequence* of them still is not - two threads can miss on
the same key and both store a value, which for a cache is wasteful, not wrong.

pop() is implemented here now instead of inheriting it: MutableMapping builds it
from __getitem__ + __delitem__, which is not atomic (two threads popping the same
key raise KeyError, see the new test), and it would dispose the value it hands to
the caller - for e.g. a cache of open files it returned a closed one.

Cost, measured with timeit: about 75 ns per operation (get 70 -> 148 ns, set 148
-> 222 ns). That is nothing next to what these caches guard - a chunk decrypt is
in the milliseconds, an item unpack in the microseconds - and a borg mount read
benchmark (1 GiB sequential, and stat+read of 200 small files, mfusepy) shows no
difference at all.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/borg/helpers/lrucache.py                | 71 +++++++++++++++------
 src/borg/testsuite/helpers/lrucache_test.py | 49 ++++++++++++++
 2 files changed, 100 insertions(+), 20 deletions(-)

diff --git a/src/borg/helpers/lrucache.py b/src/borg/helpers/lrucache.py
index d06522bd84..7f280cd5f1 100644
--- a/src/borg/helpers/lrucache.py
+++ b/src/borg/helpers/lrucache.py
@@ -1,10 +1,13 @@
+import threading
 from collections import OrderedDict
 from collections.abc import Callable, ItemsView, Iterator, KeysView, MutableMapping, ValuesView
-from typing import TypeVar
+from typing import Any, TypeVar
 
 K = TypeVar("K")
 V = TypeVar("V")
 
+_MISSING = object()
+
 
 class LRUCache(MutableMapping[K, V]):
     """
@@ -13,6 +16,16 @@ class LRUCache(MutableMapping[K, V]):
     A value that leaves the cache - evicted, deleted, or replaced by another value under
     the same key - is passed to *dispose* first, so that a cache of e.g. open files can
     close them. Use replace() to update an entry while keeping its old value alive.
+
+    Thread safety: getting, setting, deleting, popping, replacing and clearing are each
+    atomic, so several threads can share one cache without corrupting it. A *sequence* of
+    them is not: two threads can miss on the same key and both compute and store a value,
+    which for a cache is wasteful, not wrong - and the mixin methods built on top of the
+    above (setdefault, popitem, update) inherit that. Iterating (also via keys(), values(),
+    items()) yields a live view of the underlying dict, so it needs external
+    synchronization if another thread may write meanwhile.
+
+    *dispose* is called while the lock is held, so it must not use the cache.
     """
 
     _cache: OrderedDict[K, V]
@@ -25,26 +38,30 @@ def __init__(self, capacity: int, dispose: Callable[[V], None] = lambda _: None)
         self._cache = OrderedDict()
         self._capacity = capacity
         self._dispose = dispose
+        self._lock = threading.Lock()
 
     def __setitem__(self, key: K, value: V) -> None:
-        try:
-            previous = self._cache.pop(key)
-        except KeyError:
-            pass
-        else:
-            # the old value is no longer in the cache, so it is disposed like a deleted one.
-            if previous is not value:
-                self._dispose(previous)
-        while len(self._cache) >= self._capacity:
-            self._dispose(self._cache.popitem(last=False)[1])
-        self._cache[key] = value  # add the new (or refreshed) entry at the end
+        with self._lock:
+            try:
+                previous = self._cache.pop(key)
+            except KeyError:
+                pass
+            else:
+                # the old value is no longer in the cache, so it is disposed like a deleted one.
+                if previous is not value:
+                    self._dispose(previous)
+            while len(self._cache) >= self._capacity:
+                self._dispose(self._cache.popitem(last=False)[1])
+            self._cache[key] = value  # add the new (or refreshed) entry at the end
 
     def __getitem__(self, key: K) -> V:
-        self._cache.move_to_end(key)  # raise KeyError if not found
-        return self._cache[key]
+        with self._lock:
+            self._cache.move_to_end(key)  # raise KeyError if not found
+            return self._cache[key]
 
     def __delitem__(self, key: K) -> None:
-        self._dispose(self._cache.pop(key))
+        with self._lock:
+            self._dispose(self._cache.pop(key))
 
     def __contains__(self, key: object) -> bool:
         return key in self._cache
@@ -52,6 +69,18 @@ def __contains__(self, key: object) -> bool:
     def __len__(self) -> int:
         return len(self._cache)
 
+    def pop(self, key: K, default: Any = _MISSING) -> Any:
+        """Remove *key* and return its value, without disposing it."""
+        # MutableMapping would implement this as __getitem__ + __delitem__, which is not
+        # atomic (and would dispose the value).
+        with self._lock:
+            try:
+                return self._cache.pop(key)
+            except KeyError:
+                if default is _MISSING:
+                    raise
+                return default
+
     def replace(self, key: K, value: V) -> None:
         """Replace the value of an entry that is present, without disposing the old value.
 
@@ -61,13 +90,15 @@ def replace(self, key: K, value: V) -> None:
         is in the LRU order, i.e. it does not count as a use. Everything else should just
         assign to the cache.
         """
-        assert key in self._cache, "Unexpected attempt to update a non-existing item."
-        self._cache[key] = value
+        with self._lock:
+            assert key in self._cache, "Unexpected attempt to update a non-existing item."
+            self._cache[key] = value
 
     def clear(self) -> None:
-        for value in self._cache.values():
-            self._dispose(value)
-        self._cache.clear()
+        with self._lock:
+            for value in self._cache.values():
+                self._dispose(value)
+            self._cache.clear()
 
     def __iter__(self) -> Iterator[K]:
         return iter(self._cache)
diff --git a/src/borg/testsuite/helpers/lrucache_test.py b/src/borg/testsuite/helpers/lrucache_test.py
index e152a3189f..1462098bfa 100644
--- a/src/borg/testsuite/helpers/lrucache_test.py
+++ b/src/borg/testsuite/helpers/lrucache_test.py
@@ -1,3 +1,5 @@
+import sys
+import threading
 from tempfile import TemporaryFile
 
 import pytest
@@ -81,3 +83,50 @@ def test_dispose(self):
         c.clear()
         assert c.items() == set()
         assert f3.closed
+
+    def test_pop(self):
+        disposed = []
+        c = LRUCache(2, dispose=disposed.append)
+        c["a"] = 1
+        assert c.pop("a") == 1  # pop hands the value to the caller, so it is not disposed
+        assert disposed == []
+        assert "a" not in c
+        assert c.pop("a", "default") == "default"
+        with pytest.raises(KeyError):
+            c.pop("a")
+
+    def test_threaded_access(self):
+        # several threads sharing one cache must not corrupt it or raise: they hit the same
+        # keys, so they race on inserting, refreshing, evicting and removing the same entries.
+        # e.g. popping via MutableMapping (get, then delete) raises KeyError here when
+        # another thread removes the key in between.
+        c: LRUCache = LRUCache(8)
+        keys = list(range(16))  # more keys than capacity, so entries keep getting evicted
+        errors: list[Exception] = []
+        start = threading.Barrier(8)
+
+        def worker(n):
+            start.wait()
+            try:
+                for _round in range(2000):
+                    for key in keys:
+                        c[key] = n
+                        c.get(key)
+                        c.pop(key, None)
+            except Exception as e:
+                errors.append(e)
+
+        # the default switch interval (5 ms) is longer than these loops, so threads would
+        # hardly ever switch inside one cache operation and the races would not show up.
+        previous_interval = sys.getswitchinterval()
+        sys.setswitchinterval(1e-6)
+        try:
+            threads = [threading.Thread(target=worker, args=(n,)) for n in range(8)]
+            for t in threads:
+                t.start()
+            for t in threads:
+                t.join()
+        finally:
+            sys.setswitchinterval(previous_interval)
+        assert errors == []
+        assert len(c) <= 8

From 2ec4e877d73e881246032a7015f1a1a9572f9cd3 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 5 Aug 2026 17:23:44 +0530
Subject: [PATCH 022/144] check: drop obsolete --max-duration guard, credit
 oldest-first ordering in docs

The sort provides partial-check progress on its own, so --max-duration no
longer requires --max-age; bare --max-duration works as documented again.
Credit the least-recently-checked ordering (not --max-age) for resumption in
the epilog and data-structures.rst, and rework the partial-ordering test to
use intact packs so it exercises the skip path together with the sort.
---
 docs/internals/data-structures.rst            |  9 ++--
 src/borg/archiver/check_cmd.py                | 30 +++++------
 src/borg/testsuite/archiver/check_cmd_test.py | 15 +++---
 src/borg/testsuite/repository_test.py         | 50 ++++++++-----------
 4 files changed, 45 insertions(+), 59 deletions(-)

diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index 048a798500..63d0e1aa6b 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -38,10 +38,11 @@ cache/
   checked-packs
     repository check results (pack id -> timestamp, result), as a hashtable with an
     appended integrity hash. Records are kept across checks: ``check --max-age``
-    skips packs whose intact record is younger than the given age, which also lets
-    partial checks (``--max-duration``) continue where a previous one stopped.
-    Records of corrupt packs are kept for repair and always re-verified. Records of
-    packs no longer listed in packs/ are pruned when a check finishes.
+    skips packs whose intact record is younger than the given age, and partial checks
+    (``--max-duration``) verify the least-recently-checked packs first so repeated
+    runs cover the whole repository. Records of corrupt packs are kept for repair and
+    always re-verified. Records of packs no longer listed in packs/ are pruned when a
+    check finishes.
 
 There is a list of pointers to archive objects in this directory:
 
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 4118825170..36e3ceda1b 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -44,8 +44,7 @@ def do_check(self, args, repository):
         if args.repo_only and args.find_lost_archives:
             raise CommandError("--repository-only contradicts the --find-lost-archives option.")
         # resolve the marker (e.g. 4w, 12m) to seconds; calendar units (m, y) count from now, as for
-        # --older/--newer. the validator returns a string (truthy even for a zero span like 0d), so
-        # the --max-duration guard below tests max_age, the resolved value, not args.max_age.
+        # --older/--newer. max_age stays 0 when --max-age is not given, which disables reuse.
         if args.max_age is not None:
             now = archive_ts_now()
             max_age = int((now - calculate_relative_offset(args.max_age, now, earlier=True)).total_seconds())
@@ -60,10 +59,6 @@ def do_check(self, args, repository):
         if args.archives_only and args.max_age is not None:
             # --max-age only affects the repository check; --archives-only skips it.
             raise CommandError("--archives-only does not allow the --max-age option.")
-        if args.max_duration and not max_age:
-            # a partial check advances by skipping packs younger than max_age, so it needs a nonzero
-            # window (--max-age=0d resolves to 0).
-            raise CommandError("--max-duration requires the --max-age option, e.g. --max-age=4w.")
         if args.max_duration and not args.repo_only:
             # --max-duration limits only the repository check; the archives check has no max_duration
             # support.
@@ -148,18 +143,17 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         affects only the repository check and cannot be combined with
         ``--archives-only`` or ``--repair``.
 
-        The ``--max-duration`` option can be used to split a long-running repository
-        check into multiple partial checks. After the given number of seconds, the
-        check is interrupted. Because a verified pack's result is recorded and
-        reused, ``--max-duration`` requires ``--max-age``: the next partial check
-        skips the recently verified packs and continues with the rest, until every
-        pack has a result younger than ``--max-age``. Assuming a complete check
-        would take 7 hours, then running a daily check with ``--max-duration=3600
-        --max-age=1w`` (1 hour) would result in one full repository verification
-        per week. With partial repository checks you can run neither archive
-        checks, nor enable repair mode. Consequently, if you want to use
-        ``--max-duration`` you must also pass ``--repository-only``, and must not
-        pass ``--archives-only``, nor ``--repair``.
+        The ``--max-duration`` option splits a long-running repository check into
+        several partial checks. After the given number of seconds, the check is
+        interrupted. A partial check verifies the least-recently-checked packs first,
+        so repeated runs cover the whole repository. Add ``--max-age`` to also skip
+        packs whose result is still younger than the given age: once every pack has a
+        recent result, further runs re-check each pack about once per ``--max-age``.
+        Assuming a complete check would take 7 hours, running a daily check with
+        ``--max-duration=3600 --max-age=1w`` (1 hour) results in one full repository
+        verification per week. Partial repository checks run neither archive checks
+        nor repair mode, so ``--max-duration`` requires ``--repository-only`` and
+        cannot be combined with ``--archives-only`` or ``--repair``.
 
         **Warning:** Please note that partial repository checks (i.e., running with
         ``--max-duration``) can only perform non-cryptographic checksum checks on the
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index b4bad53dc4..124235c135 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -78,15 +78,13 @@ def test_check_max_age(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
 
-    # --repair and --archives-only do not allow --max-age, and --max-duration requires --max-age.
-    # a zero span like 0d resolves to max_age=0 (no reuse), so it must not satisfy the guards either.
+    # --repair and --archives-only do not allow --max-age; 0d is a valid value (resolves to no reuse).
+    # --max-duration needs --repository-only, but not --max-age: a partial check advances on its own.
     if archiver.FORK_DEFAULT:
         cmd(archiver, "check", "--repair", "--max-age=1d", exit_code=CommandError().exit_code)
         cmd(archiver, "check", "--repair", "--max-age=0d", exit_code=CommandError().exit_code)
         cmd(archiver, "check", "--archives-only", "--max-age=1d", exit_code=CommandError().exit_code)
-        cmd(archiver, "check", "--repository-only", "--max-duration=3600", exit_code=CommandError().exit_code)
-        ec = CommandError().exit_code
-        cmd(archiver, "check", "--repository-only", "--max-duration=3600", "--max-age=0d", exit_code=ec)
+        cmd(archiver, "check", "--max-duration=3600", exit_code=CommandError().exit_code)
     else:
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--repair", "--max-age=1d")
@@ -95,9 +93,10 @@ def test_check_max_age(archivers, request):
         with pytest.raises(CommandError):
             cmd(archiver, "check", "--archives-only", "--max-age=1d")
         with pytest.raises(CommandError):
-            cmd(archiver, "check", "--repository-only", "--max-duration=3600")
-        with pytest.raises(CommandError):
-            cmd(archiver, "check", "--repository-only", "--max-duration=3600", "--max-age=0d")
+            cmd(archiver, "check", "--max-duration=3600")
+
+    # a partial check runs without --max-age.
+    cmd(archiver, "check", "--repository-only", "--max-duration=3600", exit_code=0)
 
     # a check records its results, a later one with --max-age reuses them.
     output = cmd(archiver, "check", "-v", "--repository-only", exit_code=0)
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 8ca323ebab..6f26f6935e 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1083,8 +1083,9 @@ def spy_hash(key):
     return hashed_keys
 
 
-def _store_intact_pack(repository):
-    intact = fchunk(b"INTACT", chunk_id=H(1))
+def _store_intact_pack(repository, chunk_id=H(1)):
+    # a distinct chunk_id yields distinct bytes and thus a distinct sha256 pack id.
+    intact = fchunk(b"INTACT", chunk_id=chunk_id)
     intact_id = sha256(intact).digest()
     pack_key = "packs/" + bin_to_hex(intact_id)
     repository.store_store(pack_key, intact)
@@ -1414,43 +1415,34 @@ def test_check_max_age_partial_progress(tmp_path, monkeypatch):
         assert pack_a_id in after.table and pack_b_id in after.table
 
 
-def test_check_partial_verifies_least_recently_checked_first(tmp_path, monkeypatch):
-    # a partial check verifies the least-recently-checked packs first. with a budget for one pack, the
-    # pack with the oldest recorded result is verified, though its id sorts last.
+def test_check_partial_orders_stale_oldest_first_and_skips_fresh(tmp_path, monkeypatch):
+    # a partial check skips packs whose intact record is younger than max_age and verifies the rest
+    # least-recently-checked first. the sort orders by recorded time, not pack id: the older record is
+    # on the pack whose id sorts later, so an id-ordered scan would verify the two stale packs in the
+    # other order.
     max_age = 3600
-    recent_id = b"\x00" * 32  # sorts first by id
-    older_id = b"\xff" * 32  # sorts last by id
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
-        for pid in (recent_id, older_id):
-            repository.store_store("packs/" + bin_to_hex(pid), b"content-" + pid[:4])
+        fresh_id, fresh_key = _store_intact_pack(repository, chunk_id=H(1))
+        id2, _ = _store_intact_pack(repository, chunk_id=H(2))
+        id3, _ = _store_intact_pack(repository, chunk_id=H(3))
+        older_id, newer_id = (id2, id3) if id2 > id3 else (id3, id2)
+        older_key = "packs/" + bin_to_hex(older_id)
+        newer_key = "packs/" + bin_to_hex(newer_id)
 
         now = int(time.time())
         tracker = PackTracker.new(repository.store)
-        # both records are past max_age, so both need re-verification; the more recently recorded pack
-        # has the id that sorts first.
-        tracker.table[recent_id] = PackTracker.Entry(timestamp=now - (max_age + 100), result=1)
-        tracker.table[older_id] = PackTracker.Entry(timestamp=now - (max_age + 100000), result=1)
+        tracker.table[fresh_id] = PackTracker.Entry(timestamp=now - 10, result=1)  # within max_age
+        tracker.table[older_id] = PackTracker.Entry(timestamp=now - (max_age + 1000), result=1)
+        tracker.table[newer_id] = PackTracker.Entry(timestamp=now - (max_age + 100), result=1)
         tracker.save()
 
-        # freeze the clock and jump it past max_duration after the first hash, so exactly one pack is
-        # verified this run.
-        clock = {"t": 0.0}
-        monkeypatch.setattr(time, "monotonic", lambda: clock["t"])
-        hashed_keys = []
-        orig_hash = repository.store.hash
-
-        def hash_and_advance(key):
-            hashed_keys.append(key)
-            result = orig_hash(key)
-            clock["t"] = 10**9
-            return result
-
-        monkeypatch.setattr(repository.store, "hash", hash_and_advance)
+        hashed_keys = _spy_hash(repository, monkeypatch)
 
-        repository.check(repair=False, max_duration=1, max_age=max_age)
+        assert repository.check(repair=False, max_duration=3600, max_age=max_age) is True
 
         pack_hashes = [key for key in hashed_keys if key.startswith("packs/")]
-        assert pack_hashes == ["packs/" + bin_to_hex(older_id)]  # the oldest recorded pack
+        assert fresh_key not in pack_hashes  # skipped, record younger than max_age
+        assert pack_hashes == [older_key, newer_key]  # oldest record first, ignoring id order
 
 
 def test_check_max_age_reuses_records_of_plain_check(tmp_path, monkeypatch):

From 0d6fffb304bfde17282f105693b9654d95eeaca9 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 5 Aug 2026 14:21:44 +0200
Subject: [PATCH 023/144] unify the 3 archive-as-filesystem implementations,
 fixes #10020

borg materialized an archive as a browsable tree in three independent places:
fuse.py (llfuse/pyfuse3, low-level FUSE), hlfuse.py (mfusepy, high-level FUSE)
and webdav.py (ArchiveVFS + WebDAV/HTTP server). All three re-implemented tree
building, hardlink handling, the versions view, uid/gid/mode/time mapping and
reading file content from chunk lists - so every behaviour fix had to be applied
N times (e.g. the ACL/xattr exposure fix #9954 touched both FUSE variants).

New module vfs.py has that logic exactly once:

- ArchiveVFS: archive selection and name deduplication, lazily built per-archive
  trees, the versions view, hardlinks (nodes sharing one inode), item storage
  (msgpacked, path-less, as hlfuse did it), attribute mapping, xattrs/ACLs.
- DataReader: reads byte ranges out of chunk lists, with the decrypted-chunk
  cache (BORG_MOUNT_DATA_CACHE_ENTRIES) and the sequential-read position hint.
- parse_mount_options(): the "borg mount -o ..." parsing both mounts duplicated.

fuse.py, hlfuse.py and webdav.py are now thin protocol adapters over it (2604 ->
2009 lines in total). Behaviour changes that fell out of the unification:

- webdav reads now go through DownloadPipeline.fetch_many(), so the all-zero
  chunk shortcut and the parsed-chunk cache (#1678) apply to mounts as well.
- the mounts get webdav's Unicode NFC lookup fallback (macOS decomposes names).
- directories report st_nlink >= 2 (hlfuse behaviour) in both mounts.
- a chunk that is read to its end is no longer put into the data cache, so a
  full download does not evict the chunks partial (range) reads need - this was
  the FUSE behaviour, now webdav shares it.

Threading: mfusepy serves the FUSE requests of one mount from a thread pool and
webdav one request per thread, so the VFS is used from several threads. It
serializes the repository access (borgstore connections are not thread-safe) and
leaves the caches to LRUCache, which is thread-safe itself. Two threads can miss
on the same inode and both unpack the item, or store a read position over each
other - they get equal items, and the position is only a hint.

The ACL emulation and the NFC lookup are now tested against the core
(testsuite/vfs_test.py, no FUSE dependency); fuse_test.py keeps testing what is
left in the adapters: the errno mapping.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/borg/archiver/mount_cmds.py               |  30 +-
 src/borg/fuse.py                              | 722 ++---------------
 src/borg/hlfuse.py                            | 763 +++---------------
 .../testsuite/archiver/webdav_cmd_test.py     |  57 +-
 src/borg/testsuite/fuse_test.py               | 116 ++-
 src/borg/testsuite/vfs_test.py                | 114 +++
 src/borg/vfs.py                               | 723 +++++++++++++++++
 src/borg/webdav.py                            | 418 +++-------
 8 files changed, 1205 insertions(+), 1738 deletions(-)
 create mode 100644 src/borg/testsuite/vfs_test.py
 create mode 100644 src/borg/vfs.py

diff --git a/src/borg/archiver/mount_cmds.py b/src/borg/archiver/mount_cmds.py
index 020bc9099b..162b053e4f 100644
--- a/src/borg/archiver/mount_cmds.py
+++ b/src/borg/archiver/mount_cmds.py
@@ -36,27 +36,17 @@ def _do_mount(self, args, repository, manifest):
         from ..fuse_impl import has_mfusepy
 
         if has_mfusepy:
-            # Use mfusepy implementation
-            from ..hlfuse import borgfs
-
-            operations = borgfs(manifest, args, repository)
-            logger.info("Mounting filesystem")
-            try:
-                operations.mount(args.mountpoint, args.options, args.foreground, args.show_rc)
-            except RuntimeError:
-                # Relevant error message already printed to stderr by FUSE
-                raise RTError("FUSE mount failed")
+            from ..hlfuse import borgfs as fuse_operations  # high-level FUSE API
         else:
-            # Use llfuse/pyfuse3 implementation
-            from ..fuse import FuseOperations
-
-            operations = FuseOperations(manifest, args, repository)
-            logger.info("Mounting filesystem")
-            try:
-                operations.mount(args.mountpoint, args.options, args.foreground, args.show_rc)
-            except RuntimeError:
-                # Relevant error message already printed to stderr by FUSE
-                raise RTError("FUSE mount failed")
+            from ..fuse import FuseOperations as fuse_operations  # low-level FUSE API
+
+        operations = fuse_operations(manifest, args, repository)
+        logger.info("Mounting filesystem")
+        try:
+            operations.mount(args.mountpoint, args.options, args.foreground, args.show_rc)
+        except RuntimeError:
+            # Relevant error message already printed to stderr by FUSE
+            raise RTError("FUSE mount failed")
 
     def do_umount(self, args):
         """Unmounts the FUSE filesystem."""
diff --git a/src/borg/fuse.py b/src/borg/fuse.py
index c61740d2f5..bd185360ad 100644
--- a/src/borg/fuse.py
+++ b/src/borg/fuse.py
@@ -1,5 +1,9 @@
 """
-FUSE filesystem implementation for `borg mount`.
+``borg mount`` using llfuse / pyfuse3, the low-level (inode based) FUSE 2 / FUSE 3 API.
+
+This is a protocol adapter only: what an archive looks like as a file system is
+defined in vfs.py, here we just translate between that and the llfuse operations
+interface (inode numbers and EntryAttributes in, FUSEErrors out).
 
 IMPORTANT
 =========
@@ -21,20 +25,11 @@
 
 import errno
 import functools
-import io
 import os
-import stat
-import struct
-import sys
-import tempfile
 import threading
-import time
-from collections import defaultdict, Counter
 from signal import SIGINT
 from typing import TYPE_CHECKING
 
-from .constants import ROBJ_FILE_STREAM, ROBJ_DONTCARE, zeros
-
 if TYPE_CHECKING:
     # For type checking, assume llfuse is available
     # This allows mypy to understand llfuse.Operations
@@ -64,19 +59,11 @@ def async_wrapper(fn):
 
 logger = create_logger()
 
-from .crypto.low_level import blake2b_128
-from .archiver._common import build_matcher, build_filter
-from .archive import Archive, get_item_uid_gid
-from .hashindex import FuseVersionsIndex
-from .helpers import daemonizing, signal_handler, format_file_size, bin_to_hex, Error
-from .helpers import HardLinkManager
-from .helpers import msgpack
+from .helpers import daemonizing, signal_handler
 from .storelocking import LockRefresher
-from .helpers.lrucache import LRUCache
-from .item import Item
-from .platform import uid2user, gid2group, acl_text_to_xattr
-from .platformflags import is_darwin, is_linux
-from .repository import Repository
+from .vfs import ArchiveVFS, ChunkMissing, parse_mount_options
+
+BLOCK_SIZE = 512  # Standard filesystem block size for st_blocks and statfs
 
 
 def fuse_main():
@@ -93,525 +80,30 @@ def fuse_main():
         return llfuse.main(workers=1)
 
 
-# on Linux, the kernel exposes POSIX ACLs via these special, binary encoded xattrs.
-# maps the xattr name to the borg item attribute holding the ACL text.
-# empty on platforms we can not do this for, so the mount just does not offer these xattrs there.
-ACL_XATTRS = {b"system.posix_acl_access": "acl_access", b"system.posix_acl_default": "acl_default"} if is_linux else {}
-
-# size of some LRUCaches (1 element per simultaneously open file)
-# note: _inode_cache might have rather large elements - Item.chunks can be large!
-#       also, simultaneously reading too many files should be avoided anyway.
-#       thus, do not set FILES to high values.
-FILES = 4
-
-
-class ItemCache:
-    """
-    This is the "meat" of the filesystem's metadata storage.
-
-    This class generates inode numbers that efficiently index items in archives
-    and retrieves items from these inode numbers.
-    """
-
-    # 2 MiB are approximately ~230000 items (depends on the average number of items per metadata chunk).
-    #
-    # Since growing a bytearray has to copy it, growing it will converge to O(n^2), however,
-    # this is not yet relevant due to the swiftness of copying memory. If it becomes an issue,
-    # use an anonymous mmap and just resize that (or, if on 64 bit, make it so big you never need
-    # to resize it in the first place; that's free).
-    GROW_META_BY = 2 * 1024 * 1024
-
-    indirect_entry_struct = struct.Struct("=cII")
-    assert indirect_entry_struct.size == 9
-
-    def __init__(self, repository, repo_objs):
-        self.repository = repository
-        self.repo_objs = repo_objs
-        # self.meta, the "meta-array" is a densely packed array of metadata about where items can be found.
-        # It is indexed by the inode number minus self.offset. (This is in a way eerily similar to how the first
-        # unices did this).
-        # The meta-array contains chunk IDs and item entries (described in iter_archive_items).
-        # The chunk IDs are referenced by item entries through relative offsets,
-        # which are bounded by the metadata chunk size.
-        self.meta = bytearray()
-        # The current write offset in self.meta
-        self.write_offset = 0
-
-        # Offset added to meta-indices, resulting in inodes,
-        # or subtracted from inodes, resulting in meta-indices.
-        # XXX: Merge FuseOperations.items and ItemCache to avoid
-        #      this implicit limitation / hack (on the number of synthetic inodes, degenerate
-        #      cases can inflate their number far beyond the number of archives).
-        self.offset = 1000000
-
-        # A temporary file that contains direct items, i.e. items directly cached in this layer.
-        # These are items that span more than one chunk and thus cannot be efficiently cached
-        # by the object cache (self.chunks), which would require variable-length structures;
-        # possible but not worth the effort, see iter_archive_items.
-        self.fd = tempfile.TemporaryFile(prefix="borg-tmp")
-
-        # A small LRU cache for chunks requested by ItemCache.get() from the object cache,
-        # this significantly speeds up directory traversal and similar operations which
-        # tend to re-read the same chunks over and over.
-        # The capacity is kept low because increasing it does not provide any significant advantage,
-        # but makes LRUCache's square behaviour noticeable and consumes more memory.
-        self.chunks = LRUCache(capacity=10)
-
-        # Instrumentation
-        # Count of indirect items, i.e. data is cached in the object cache, not directly in this cache
-        self.indirect_items = 0
-        # Count of direct items, i.e. data is in self.fd
-        self.direct_items = 0
-
-    def get(self, inode):
-        offset = inode - self.offset
-        if offset < 0:
-            raise ValueError("ItemCache.get() called with an invalid inode number")
-        if self.meta[offset] == ord(b"I"):
-            _, chunk_id_relative_offset, chunk_offset = self.indirect_entry_struct.unpack_from(self.meta, offset)
-            chunk_id_offset = offset - chunk_id_relative_offset
-            # bytearray slices are bytearrays as well, explicitly convert to bytes()
-            chunk_id = bytes(self.meta[chunk_id_offset : chunk_id_offset + 32])
-            chunk = self.chunks.get(chunk_id)
-            if not chunk:
-                cdata = self.repository.get(chunk_id)
-                _, chunk = self.repo_objs.parse(chunk_id, cdata, ro_type=ROBJ_DONTCARE)
-                self.chunks[chunk_id] = chunk
-            data = memoryview(chunk)[chunk_offset:]
-            unpacker = msgpack.Unpacker()
-            unpacker.feed(data)
-            return Item(internal_dict=next(unpacker))
-        elif self.meta[offset] == ord(b"S"):
-            fd_offset = int.from_bytes(self.meta[offset + 1 : offset + 9], "little")
-            self.fd.seek(fd_offset, io.SEEK_SET)
-            return Item(internal_dict=next(msgpack.Unpacker(self.fd, read_size=1024)))
-        else:
-            raise ValueError("Invalid entry type in self.meta")
-
-    def iter_archive_items(self, archive_item_ids, filter=None):
-        unpacker = msgpack.Unpacker()
-
-        # Current offset in the metadata stream, which consists of all metadata chunks glued together
-        stream_offset = 0
-        # Offset of the current chunk in the metadata stream
-        chunk_begin = 0
-        # Length of the chunk preceding the current chunk
-        last_chunk_length = 0
-        msgpacked_bytes = b""
-
-        write_offset = self.write_offset
-        meta = self.meta
-        pack_indirect_into = self.indirect_entry_struct.pack_into
-
-        for key, cdata in zip(archive_item_ids, self.repository.get_many(archive_item_ids)):
-            _, data = self.repo_objs.parse(key, cdata, ro_type=ROBJ_DONTCARE)
-            # Store the chunk ID in the meta-array
-            if write_offset + 32 >= len(meta):
-                meta.extend(bytes(self.GROW_META_BY))
-            meta[write_offset : write_offset + 32] = key
-            current_id_offset = write_offset
-            write_offset += 32
-
-            chunk_begin += last_chunk_length
-            last_chunk_length = len(data)
-
-            unpacker.feed(data)
-            while True:
-                try:
-                    item = unpacker.unpack()
-                    need_more_data = False
-                except msgpack.OutOfData:
-                    need_more_data = True
-
-                start = stream_offset - chunk_begin
-                # tell() is not helpful for the need_more_data case, but we know it is the remainder
-                # of the data in that case. in the other case, tell() works as expected.
-                length = (len(data) - start) if need_more_data else (unpacker.tell() - stream_offset)
-                msgpacked_bytes += data[start : start + length]
-                stream_offset += length
-
-                if need_more_data:
-                    # Need more data, feed the next chunk
-                    break
-
-                item = Item(internal_dict=item)
-                if filter and not filter(item):
-                    msgpacked_bytes = b""
-                    continue
-
-                current_item = msgpacked_bytes
-                current_item_length = len(current_item)
-                current_spans_chunks = stream_offset - current_item_length < chunk_begin
-                msgpacked_bytes = b""
-
-                if write_offset + 9 >= len(meta):
-                    meta.extend(bytes(self.GROW_META_BY))
-
-                # item entries in the meta-array come in two different flavours, both nine bytes long.
-                # (1) for items that span chunks:
-                #
-                #     'S' + 8 byte offset into the self.fd file, where the msgpacked item starts.
-                #
-                # (2) for items that are completely contained in one chunk, which usually is the great majority
-                #     (about 700:1 for system backups)
-                #
-                #     'I' + 4 byte offset where the chunk ID is + 4 byte offset in the chunk
-                #     where the msgpacked items starts
-                #
-                #     The chunk ID offset is the number of bytes _back_ from the start of the entry, i.e.:
-                #
-                #     |Chunk ID| ....          |S1234abcd|
-                #      ^------ offset ----------^
-
-                if current_spans_chunks:
-                    pos = self.fd.seek(0, io.SEEK_END)
-                    self.fd.write(current_item)
-                    meta[write_offset : write_offset + 9] = b"S" + pos.to_bytes(8, "little")
-                    self.direct_items += 1
-                else:
-                    item_offset = stream_offset - current_item_length - chunk_begin
-                    pack_indirect_into(meta, write_offset, b"I", write_offset - current_id_offset, item_offset)
-                    self.indirect_items += 1
-                inode = write_offset + self.offset
-                write_offset += 9
-
-                yield inode, item
-
-        self.write_offset = write_offset
-
-
-class FuseBackend:
-    """Virtual filesystem based on archive(s) to provide information to fuse"""
+class FuseOperations(llfuse.Operations):
+    """Export archive contents as a FUSE filesystem"""
 
     def __init__(self, manifest, args, repository):
-        self._args = args
-        self.numeric_ids = args.numeric_ids
+        llfuse.Operations.__init__(self)
         self._manifest = manifest
-        self.repo_objs = manifest.repo_objs
-        self.repository_uncached = manifest.repository
-        # Maps inode numbers to Item instances. This is used for synthetic inodes, i.e. file-system objects that are
-        # made up and are not contained in the archives. For example archive directories or intermediate directories
-        # not contained in archives.
-        self._items = {}
-        # cache up to <FILES> Items
-        self._inode_cache = LRUCache(capacity=FILES)
-        # _inode_count is the current count of synthetic inodes, i.e. those in self._items
-        self.inode_count = 0
-        # Maps inode numbers to the inode number of the parent
-        self.parent = {}
-        # Maps inode numbers to a dictionary mapping byte directory entry names to their inode numbers,
-        # i.e. this contains all dirents of everything that is mounted. (It becomes really big).
-        self.contents = defaultdict(dict)
-        self.default_uid = os.getuid()
-        self.default_gid = os.getgid()
-        self.default_dir = None
-        # Archives to be loaded when first accessed, mapped by their placeholder inode
-        self.pending_archives = {}
-        self.cache = ItemCache(repository, self.repo_objs)
+        self._args = args
+        self._repository = repository
         # serializes all repository access (FUSE handlers and the background lock-refresh
-        # thread), because borgstore connections are not thread-safe. see _lock_refresh.
+        # thread), because borgstore connections are not thread-safe.
         self._repo_lock = threading.RLock()
-        self.allow_damaged_files = False
-        self.versions = False
-        self.uid_forced = None
-        self.gid_forced = None
-        self.umask = 0
-        self.archive_root_dir = {}  # archive ID --> directory name
-
-    def _create_filesystem(self):
-        self._create_dir(parent=1)  # first call, create root dir (inode == 1)
-        self.versions_index = FuseVersionsIndex()
-        archives = self._manifest.archives.list_considering(self._args)
-        name_counter = Counter(a.name for a in archives)
-        duplicate_names = {a.name for a in archives if name_counter[a.name] > 1}
-        for archive in archives:
-            name = f"{archive.name}"
-            if name in duplicate_names:
-                name += f"-{bin_to_hex(archive.id):.8}"
-            self.archive_root_dir[archive.id] = name
-        for archive in archives:
-            if self.versions:
-                # process archives immediately
-                self._process_archive(archive.id)
-            else:
-                # lazily load archives, create archive placeholder inode
-                archive_inode = self._create_dir(parent=1, mtime=int(archive.ts.timestamp() * 1e9))
-                name = self.archive_root_dir[archive.id]
-                self.contents[1][os.fsencode(name)] = archive_inode
-                self.pending_archives[archive_inode] = archive
-
-    def get_item(self, inode):
-        item = self._inode_cache.get(inode)
-        if item is not None:
-            return item
-        try:
-            # this is a cheap get-from-dictionary operation, no need to cache the result.
-            return self._items[inode]
-        except KeyError:
-            # while self.cache does some internal caching, it has still quite some overhead, so we cache the result.
-            with self._repo_lock:
-                item = self.cache.get(inode)
-            self._inode_cache[inode] = item
-            return item
-
-    def check_pending_archive(self, inode):
-        # Check if this is an archive we need to load
-        archive_info = self.pending_archives.pop(inode, None)
-        if archive_info is not None:
-            with self._repo_lock:
-                self._process_archive(archive_info.id, [os.fsencode(self.archive_root_dir[archive_info.id])])
-
-    def _allocate_inode(self):
-        self.inode_count += 1
-        return self.inode_count
-
-    def _create_dir(self, parent, mtime=None):
-        """Create directory"""
-        ino = self._allocate_inode()
-        if mtime is not None:
-            self._items[ino] = Item(internal_dict=self.default_dir.as_dict())
-            self._items[ino].mtime = mtime
-        else:
-            self._items[ino] = self.default_dir
-        self.parent[ino] = parent
-        return ino
-
-    def find_inode(self, path, prefix=[]):
-        segments = prefix + path.split(b"/")
-        inode = 1
-        for segment in segments:
-            inode = self.contents[inode][segment]
-        return inode
-
-    def _process_archive(self, archive_id, prefix=[]):
-        """Build FUSE inode hierarchy from archive metadata"""
-        self.file_versions = {}  # for versions mode: original path -> version
-        t0 = time.perf_counter()
-        archive = Archive(self._manifest, archive_id)
-        strip_components = self._args.strip_components
-        # omitting args.pattern_roots here, restricting to paths only by cli args.paths:
-        matcher = build_matcher(self._args.patterns, self._args.paths)
-        hlm = HardLinkManager(id_type=bytes, info_type=str)  # hlid -> path
-
-        filter = build_filter(matcher, strip_components)
-        for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter):
-            if strip_components:
-                item.path = os.sep.join(item.path.split(os.sep)[strip_components:])
-            path = os.fsencode(item.path)
-            is_dir = stat.S_ISDIR(item.mode)
-            if is_dir:
-                try:
-                    # This can happen if an archive was created with a command line like
-                    # $ borg create ... dir1/file dir1
-                    # In this case the code below will have created a default_dir inode for dir1 already.
-                    inode = self.find_inode(path, prefix)
-                except KeyError:
-                    pass
-                else:
-                    self._items[inode] = item
-                    continue
-            segments = prefix + path.split(b"/")
-            parent = 1
-            for segment in segments[:-1]:
-                parent = self._process_inner(segment, parent)
-            self._process_leaf(segments[-1], item, parent, prefix, is_dir, item_inode, hlm)
-        duration = time.perf_counter() - t0
-        logger.debug("fuse: _process_archive completed in %.1f s for archive %s", duration, archive.name)
-
-    def _process_leaf(self, name, item, parent, prefix, is_dir, item_inode, hlm):
-        path = item.path
-        del item.path  # save some space
-
-        def file_version(item, path):
-            if "chunks" in item:
-                file_id = blake2b_128(path)
-                current_version, previous_id = self.versions_index.get(file_id, (0, None))
-
-                contents_id = blake2b_128(b"".join(chunk_id for chunk_id, _ in item.chunks))
-
-                if contents_id != previous_id:
-                    current_version += 1
-                    self.versions_index[file_id] = current_version, contents_id
-
-                return current_version
-
-        def make_versioned_name(name, version, add_dir=False):
-            if add_dir:
-                # add intermediate directory with same name as filename
-                path_fname = name.rsplit(b"/", 1)
-                name += b"/" + path_fname[-1]
-            # keep original extension at end to avoid confusing tools
-            name, ext = os.path.splitext(name)
-            version_enc = os.fsencode(".%05d" % version)
-            return name + version_enc + ext
-
-        if "hlid" in item:
-            link_target = hlm.retrieve(id=item.hlid, default=None)
-            if link_target is not None:
-                # Hard link was extracted previously, just link
-                link_target = os.fsencode(link_target)
-                if self.versions:
-                    # adjust link target name with version
-                    version = self.file_versions[link_target]
-                    link_target = make_versioned_name(link_target, version, add_dir=True)
-                try:
-                    inode = self.find_inode(link_target, prefix)
-                except KeyError:
-                    logger.warning("Skipping broken hard link: %s -> %s", path, link_target)
-                    return
-                item = self.get_item(inode)
-                item.nlink = item.get("nlink", 1) + 1
-                self._items[inode] = item
-            else:
-                inode = item_inode
-                self._items[inode] = item
-                # remember extracted item path, so that following hard links don't extract twice.
-                hlm.remember(id=item.hlid, info=path)
-        else:
-            inode = item_inode
-
-        if self.versions and not is_dir:
-            parent = self._process_inner(name, parent)
-            enc_path = os.fsencode(path)
-            version = file_version(item, enc_path)
-            if version is not None:
-                # regular file, with contents
-                name = make_versioned_name(name, version)
-                self.file_versions[enc_path] = version
-
-        self.parent[inode] = parent
-        if name:
-            self.contents[parent][name] = inode
-
-    def _process_inner(self, name, parent_inode):
-        dir = self.contents[parent_inode]
-        if name in dir:
-            inode = dir[name]
-        else:
-            inode = self._create_dir(parent_inode)
-            if name:
-                dir[name] = inode
-        return inode
-
-
-class FuseOperations(llfuse.Operations, FuseBackend):
-    """Export archive as a FUSE filesystem"""
-
-    def __init__(self, manifest, args, repository):
-        llfuse.Operations.__init__(self)
-        FuseBackend.__init__(self, manifest, args, repository)
-        data_cache_capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1))
-        logger.debug("mount data cache capacity: %d chunks", data_cache_capacity)
-        self.data_cache = LRUCache(capacity=data_cache_capacity)
-        self._last_pos = LRUCache(capacity=FILES)
-
-    def sig_info_handler(self, sig_no, stack):
-        logger.debug(
-            "fuse: %d synth inodes, %d edges (%s)",
-            self.inode_count,
-            len(self.parent),
-            # getsizeof is the size of the dict itself; key and value are two small-ish integers,
-            # which are shared due to code structure (this has been verified).
-            format_file_size(sys.getsizeof(self.parent) + len(self.parent) * sys.getsizeof(self.inode_count)),
-        )
-        logger.debug("fuse: %d pending archives", len(self.pending_archives))
-        logger.debug(
-            "fuse: ItemCache %d entries (%d direct, %d indirect), meta-array size %s, direct items size %s",
-            self.cache.direct_items + self.cache.indirect_items,
-            self.cache.direct_items,
-            self.cache.indirect_items,
-            format_file_size(sys.getsizeof(self.cache.meta)),
-            format_file_size(os.stat(self.cache.fd.fileno()).st_size),
-        )
-        logger.debug(
-            "fuse: data cache: %d/%d entries, %s",
-            len(self.data_cache.items()),
-            self.data_cache._capacity,
-            format_file_size(sum(len(chunk) for key, chunk in self.data_cache.items())),
-        )
+        self.vfs = None  # created by mount(), once the mount options are known
 
     def mount(self, mountpoint, mount_options, foreground=False, show_rc=False):
         """Mount filesystem on *mountpoint* with *mount_options*."""
-
-        def pop_option(options, key, present, not_present, wanted_type, int_base=0):
-            assert isinstance(options, list)  # we mutate this
-            for idx, option in enumerate(options):
-                if option == key:
-                    options.pop(idx)
-                    return present
-                if option.startswith(key + "="):
-                    options.pop(idx)
-                    value = option.split("=", 1)[1]
-                    if wanted_type is bool:
-                        v = value.lower()
-                        if v in ("y", "yes", "true", "1"):
-                            return True
-                        if v in ("n", "no", "false", "0"):
-                            return False
-                        raise ValueError("unsupported value in option: %s" % option)
-                    if wanted_type is int:
-                        try:
-                            return int(value, base=int_base)
-                        except ValueError:
-                            raise ValueError("unsupported value in option: %s" % option) from None
-                    try:
-                        return wanted_type(value)
-                    except ValueError:
-                        raise ValueError("unsupported value in option: %s" % option) from None
-            else:
-                return not_present
-
-        # default_permissions enables permission checking by the kernel. Without
-        # this, any umask (or uid/gid) would not have an effect and this could
-        # cause security issues if used with allow_other mount option.
-        # When not using allow_other or allow_root, access is limited to the
-        # mounting user anyway.
-        options = ["fsname=borgfs", "ro", "default_permissions"]
-        if mount_options:
-            options.extend(mount_options.split(","))
-        if is_darwin:
-            # macFUSE supports a volname mount option to give what finder displays on desktop / in directory list.
-            volname = pop_option(options, "volname", "", "", str)
-            # if the user did not specify it, we make something up,
-            # because otherwise it would be "macFUSE Volume 0 (Python)", #7690.
-            volname = volname or f"{os.path.basename(mountpoint)} (borgfs)"
-            options.append(f"volname={volname}")
-        ignore_permissions = pop_option(options, "ignore_permissions", True, False, bool)
-        if ignore_permissions:
-            # in case users have a use-case that requires NOT giving "default_permissions",
-            # this is enabled by the custom "ignore_permissions" mount option which just
-            # removes "default_permissions" again:
-            pop_option(options, "default_permissions", True, False, bool)
-        self.allow_damaged_files = pop_option(options, "allow_damaged_files", True, False, bool)
-        self.versions = pop_option(options, "versions", True, False, bool)
-        self.uid_forced = pop_option(options, "uid", None, None, int)
-        self.gid_forced = pop_option(options, "gid", None, None, int)
-        self.umask = pop_option(options, "umask", 0, 0, int, int_base=8)  # umask is octal, e.g. 222 or 0222
-        dir_uid = self.uid_forced if self.uid_forced is not None else self.default_uid
-        dir_gid = self.gid_forced if self.gid_forced is not None else self.default_gid
-        dir_user = uid2user(dir_uid)
-        dir_group = gid2group(dir_gid)
-        if not isinstance(dir_user, str):
-            raise Error(
-                f"uid {dir_uid} can not be resolved to a username. "
-                f"Please check that the corresponding user exists or do not specify a uid mount option."
-            )
-        if not isinstance(dir_group, str):
-            raise Error(
-                f"gid {dir_gid} can not be resolved to a group name. "
-                f"Please check that the corresponding group exists or do not specify a gid mount option."
-            )
-        dir_mode = 0o40755 & ~self.umask
-        self.default_dir = Item(
-            mode=dir_mode, mtime=int(time.time() * 1e9), user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid
-        )
-        self._create_filesystem()
+        options, vfs_options = parse_mount_options(self._args, mountpoint, mount_options)
+        self.vfs = ArchiveVFS(self._manifest, self._args, self._repository, lock=self._repo_lock, options=vfs_options)
+        self.vfs.create_filesystem()
         llfuse.init(self, mountpoint, options)
         if not foreground:
             with daemonizing(show_rc=show_rc) as (old_id, new_id):
                 # the locking process' PID is changing, migrate it:
                 logger.debug("fuse: mount repo, going to background: migrating lock.")
-                self.repository_uncached.migrate_lock(old_id, new_id)
+                self._repository.migrate_lock(old_id, new_id)
 
         # If the file system crashes, we do not want to umount because in that
         # case the mountpoint suddenly appears to become empty. This can have
@@ -621,7 +113,7 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
         umount = False
         # keep the repository lock of an idle mount alive, so it is not killed as stale (see #9872).
         # started here (after a possible daemonizing fork, as threads do not survive fork()).
-        lock_refreshing_thread = LockRefresher(self.repository_uncached.info, sleep_interval=60, lock=self._repo_lock)
+        lock_refreshing_thread = LockRefresher(self._repository.info, sleep_interval=60, lock=self._repo_lock)
         lock_refreshing_thread.start()
         try:
             with signal_handler("SIGUSR1", self.sig_info_handler), signal_handler("SIGINFO", self.sig_info_handler):
@@ -632,11 +124,45 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
             lock_refreshing_thread.terminate()
             llfuse.close(umount)
 
+    def sig_info_handler(self, sig_no, stack):
+        self.vfs.log_stats()
+
+    # -- helpers ---------------------------------------------------------------
+
+    def _getattr(self, inode, ctx=None):
+        attrs = self.vfs.attrs(inode)
+        entry = llfuse.EntryAttributes()
+        entry.st_ino = inode
+        entry.generation = 0
+        entry.entry_timeout = 300
+        entry.attr_timeout = 300
+        entry.st_mode = attrs.mode
+        entry.st_nlink = attrs.nlink
+        entry.st_uid = attrs.uid
+        entry.st_gid = attrs.gid
+        entry.st_rdev = attrs.rdev
+        entry.st_size = attrs.size
+        entry.st_blksize = BLOCK_SIZE
+        entry.st_blocks = (entry.st_size + entry.st_blksize - 1) // entry.st_blksize
+        entry.st_mtime_ns = attrs.mtime_ns
+        entry.st_atime_ns = attrs.atime_ns
+        entry.st_ctime_ns = attrs.ctime_ns
+        entry.st_birthtime_ns = attrs.birthtime_ns
+        return entry
+
+    def _dir_node(self, inode):
+        try:
+            return self.vfs.get_node(inode)
+        except KeyError:
+            raise llfuse.FUSEError(errno.ENOTDIR) from None
+
+    # -- filesystem operations -------------------------------------------------
+
     @async_wrapper
     def statfs(self, ctx=None):
         stat_ = llfuse.StatvfsData()
-        stat_.f_bsize = 512  # Filesystem block size
-        stat_.f_frsize = 512  # Fragment size
+        stat_.f_bsize = BLOCK_SIZE  # Filesystem block size
+        stat_.f_frsize = BLOCK_SIZE  # Fragment size
         stat_.f_blocks = 0  # Size of fs in f_frsize units
         stat_.f_bfree = 0  # Number of free blocks
         stat_.f_bavail = 0  # Number of free blocks for unprivileged users
@@ -646,74 +172,37 @@ def statfs(self, ctx=None):
         stat_.f_namemax = 255  # == NAME_MAX (depends on archive source OS / FS)
         return stat_
 
-    def _getattr(self, inode, ctx=None):
-        item = self.get_item(inode)
-        entry = llfuse.EntryAttributes()
-        entry.st_ino = inode
-        entry.generation = 0
-        entry.entry_timeout = 300
-        entry.attr_timeout = 300
-        entry.st_mode = item.mode & ~self.umask
-        entry.st_nlink = item.get("nlink", 1)
-        entry.st_uid, entry.st_gid = get_item_uid_gid(
-            item,
-            numeric=self.numeric_ids,
-            uid_default=self.default_uid,
-            gid_default=self.default_gid,
-            uid_forced=self.uid_forced,
-            gid_forced=self.gid_forced,
-        )
-        entry.st_rdev = item.get("rdev", 0)
-        entry.st_size = item.get_size()
-        entry.st_blksize = 512
-        entry.st_blocks = (entry.st_size + entry.st_blksize - 1) // entry.st_blksize
-        # note: older archives only have mtime (not atime nor ctime)
-        entry.st_mtime_ns = mtime_ns = item.mtime
-        entry.st_atime_ns = item.get("atime", mtime_ns)
-        entry.st_ctime_ns = item.get("ctime", mtime_ns)
-        entry.st_birthtime_ns = item.get("birthtime", mtime_ns)
-        return entry
-
     @async_wrapper
     def getattr(self, inode, ctx=None):
         return self._getattr(inode, ctx=ctx)
 
     @async_wrapper
     def listxattr(self, inode, ctx=None):
-        item = self.get_item(inode)
-        names = list(item.get("xattrs", {}).keys())
-        # expose the archived POSIX ACLs, so e.g. getfacl or tools copying from the mount can read them.
-        names.extend(xattr_name for xattr_name, attr in ACL_XATTRS.items() if attr in item)
-        return names
+        return self.vfs.listxattr(inode)
 
     @async_wrapper
     def getxattr(self, inode, name, ctx=None):
-        item = self.get_item(inode)
-        if name in ACL_XATTRS:
-            acl = item.get(ACL_XATTRS[name])
-            if acl is None:
-                raise llfuse.FUSEError(ENOATTR)
-            try:
-                return acl_text_to_xattr(acl, numeric_ids=self.numeric_ids)
-            except ValueError:
-                logger.warning("mount: could not convert ACL of inode %d to the xattr representation", inode)
-                raise llfuse.FUSEError(errno.EIO) from None
         try:
-            return item.get("xattrs", {})[name] or b""
+            return self.vfs.getxattr(inode, name)
         except KeyError:
             raise llfuse.FUSEError(ENOATTR) from None
+        except ValueError:
+            logger.warning("mount: could not convert ACL of inode %d to the xattr representation", inode)
+            raise llfuse.FUSEError(errno.EIO) from None
 
     @async_wrapper
     def lookup(self, parent_inode, name, ctx=None):
-        self.check_pending_archive(parent_inode)
+        node = self._dir_node(parent_inode)
         if name == b".":
             inode = parent_inode
         elif name == b"..":
-            inode = self.parent[parent_inode]
+            inode = node.parent.ino if node.parent is not None else parent_inode
         else:
-            inode = self.contents[parent_inode].get(name)
-            if not inode:
-                raise llfuse.FUSEError(errno.ENOENT)
+            try:
+                _, child = self.vfs.lookup(node, os.fsdecode(name))
+            except KeyError:
+                raise llfuse.FUSEError(errno.ENOENT) from None
+            inode = child.ino
         return self._getattr(inode)
 
     @async_wrapper
@@ -722,69 +211,28 @@ def open(self, inode, flags, ctx=None):
 
     @async_wrapper
     def opendir(self, inode, ctx=None):
-        self.check_pending_archive(inode)
+        self.vfs.ensure_loaded(self._dir_node(inode))
         return inode
 
     @async_wrapper
     def read(self, fh, offset, size):
-        parts = []
-        item = self.get_item(fh)
-
-        # optimize for linear reads:
-        # we cache the chunk number and the in-file offset of the chunk in _last_pos[fh]
-        chunk_no, chunk_offset = self._last_pos.get(fh, (0, 0))
-        if chunk_offset > offset:
-            # this is not a linear read, so we lost track and need to start from beginning again...
-            chunk_no, chunk_offset = (0, 0)
-
-        offset -= chunk_offset
-        chunks = item.chunks
-        # note: using index iteration to avoid frequently copying big (sub)lists by slicing
-        for idx in range(chunk_no, len(chunks)):
-            id, s = chunks[idx]
-            if s < offset:
-                offset -= s
-                chunk_offset += s
-                chunk_no += 1
-                continue
-            n = min(size, s - offset)
-            if id in self.data_cache:
-                data = self.data_cache[id]
-                if offset + n == len(data):
-                    # evict fully read chunk from cache
-                    del self.data_cache[id]
-            else:
-                try:
-                    with self._repo_lock:
-                        cdata = self.repository_uncached.get(id)
-                except Repository.ObjectNotFound:
-                    if self.allow_damaged_files:
-                        data = zeros[:s]
-                        assert len(data) == s
-                    else:
-                        raise llfuse.FUSEError(errno.EIO) from None
-                else:
-                    _, data = self.repo_objs.parse(id, cdata, ro_type=ROBJ_FILE_STREAM)
-                if offset + n < len(data):
-                    # chunk was only partially read, cache it
-                    self.data_cache[id] = data
-            parts.append(data[offset : offset + n])
-            offset = 0
-            size -= n
-            if not size:
-                if fh in self._last_pos:
-                    self._last_pos.replace(fh, (chunk_no, chunk_offset))
-                else:
-                    self._last_pos[fh] = (chunk_no, chunk_offset)
-                break
-        return b"".join(parts)
+        try:
+            return self.vfs.read(fh, offset, size, pos_key=fh)
+        except ChunkMissing:
+            raise llfuse.FUSEError(errno.EIO) from None
+
+    def _readdir_entries(self, fh):
+        node = self._dir_node(fh)
+        parent = node.parent if node.parent is not None else node
+        entries = [(b".", node.ino), (b"..", parent.ino)]
+        entries.extend((os.fsencode(name), child.ino) for name, child in self.vfs.children(node))
+        return entries
 
     # note: we can't have a generator (with yield) and not a generator (async) in the same method
     if has_pyfuse3:
 
         async def readdir(self, fh, off, token):  # type: ignore[misc]
-            entries = [(b".", fh), (b"..", self.parent[fh])]
-            entries.extend(self.contents[fh].items())
+            entries = self._readdir_entries(fh)
             for i, (name, inode) in enumerate(entries[off:], off):
                 attrs = self._getattr(inode)
                 if not llfuse.readdir_reply(token, name, attrs, i + 1):
@@ -793,13 +241,11 @@ async def readdir(self, fh, off, token):  # type: ignore[misc]
     else:
 
         def readdir(self, fh, off):  # type: ignore[misc]
-            entries = [(b".", fh), (b"..", self.parent[fh])]
-            entries.extend(self.contents[fh].items())
+            entries = self._readdir_entries(fh)
             for i, (name, inode) in enumerate(entries[off:], off):
                 attrs = self._getattr(inode)
                 yield name, attrs, i + 1
 
     @async_wrapper
     def readlink(self, inode, ctx=None):
-        item = self.get_item(inode)
-        return os.fsencode(item.target)
+        return os.fsencode(self.vfs.readlink(inode))
diff --git a/src/borg/hlfuse.py b/src/borg/hlfuse.py
index 201dccca9a..242100757e 100644
--- a/src/borg/hlfuse.py
+++ b/src/borg/hlfuse.py
@@ -1,15 +1,15 @@
-import datetime
+"""
+``borg mount`` using mfusepy, the high-level (path based) FUSE 2 / FUSE 3 API.
+
+This is a protocol adapter only: what an archive looks like as a file system is
+defined in vfs.py, here we just translate between that and the mfusepy operations
+interface (paths and stat dicts in, errnos out).
+"""
+
 import errno
-import hashlib
-import os
-import stat
 import threading
-import time
-from collections import Counter
 from typing import TYPE_CHECKING
 
-from .constants import ROBJ_FILE_STREAM, zeros, ROBJ_DONTCARE
-
 if TYPE_CHECKING:
     # For type checking, assume mfusepy is available
     # This allows mypy to understand hlfuse.Operations
@@ -22,531 +22,46 @@
 
 logger = create_logger()
 
-from .archiver._common import build_matcher, build_filter
-from .archive import Archive, get_item_uid_gid
-from .hashindex import FuseVersionsIndex
-from .helpers import daemonizing, signal_handler, bin_to_hex, Error
-from .helpers import HardLinkManager
-from .helpers import msgpack
+from .helpers import daemonizing, signal_handler
 from .storelocking import LockRefresher
-from .helpers.lrucache import LRUCache
-from .item import Item
-from .platform import uid2user, gid2group, acl_text_to_xattr
-from .platformflags import is_darwin, is_linux
-from .repository import Repository
+from .vfs import ArchiveVFS, ChunkMissing, parse_mount_options
 
 BLOCK_SIZE = 512  # Standard filesystem block size for st_blocks and statfs
 
-# on Linux, the kernel exposes POSIX ACLs via these special, binary encoded xattrs.
-# maps the xattr name to the borg item attribute holding the ACL text.
-# empty on platforms we can not do this for, so the mount just does not offer these xattrs there.
-ACL_XATTRS = {"system.posix_acl_access": "acl_access", "system.posix_acl_default": "acl_default"} if is_linux else {}
-
-DEBUG_LOG: str | None = None  # os.path.join(os.getcwd(), "fuse_debug.log")
-
-
-def debug_log(msg):
-    """Append debug message to fuse_debug.log"""
-    if DEBUG_LOG:
-        timestamp = datetime.datetime.now().strftime("%H:%M:%S.%f")[:-3]
-        with open(DEBUG_LOG, "a") as f:
-            f.write(f"{timestamp} {msg}\n")
-
-
-class DirEntry:
-    __slots__ = ("ino", "parent", "children")
 
-    def __init__(self, ino, parent=None):
-        self.ino = ino  # inode number
-        self.parent = parent
-        self.children = None  # name (bytes) -> DirEntry, lazily allocated
+class borgfs(hlfuse.Operations):
+    """Export archive contents as a FUSE filesystem"""
 
-    def add_child(self, name, child):
-        """Add a child entry, lazily allocating the children dict if needed."""
-        if self.children is None:
-            self.children = {}
-        self.children[name] = child
-
-    def get_child(self, name):
-        """Get a child entry by name, returns None if not found."""
-        if self.children is None:
-            return None
-        return self.children.get(name)
-
-    def has_child(self, name):
-        """Check if a child with the given name exists."""
-        if self.children is None:
-            return False
-        return name in self.children
-
-    def iter_children(self):
-        """Iterate over (name, child) pairs."""
-        if self.children is None:
-            return iter([])
-        return self.children.items()
-
-
-class FuseBackend:
-    """Virtual filesystem based on archive(s) to provide information to fuse"""
+    use_ns = True
 
     def __init__(self, manifest, args, repository):
-        self._args = args
-        self.numeric_ids = args.numeric_ids
+        hlfuse.Operations.__init__(self)
         self._manifest = manifest
-        self.repo_objs = manifest.repo_objs
-        self.repository = repository
+        self._args = args
+        self._repository = repository
         # serializes all repository access (FUSE handlers and the background lock-refresh
-        # thread), because borgstore connections are not thread-safe. see _lock_refresh.
+        # thread), because borgstore connections are not thread-safe.
         self._repo_lock = threading.RLock()
-
-        self.default_uid = os.getuid()
-        self.default_gid = os.getgid()
-        self.default_dir = None
-
-        self.current_ino = 0
-        self.inodes = {}  # node.ino -> packed item
-        self.root = self._create_node()
-        self.pending_archives = {}  # DirEntry -> Archive
-
-        self.allow_damaged_files = False
-        self.versions = False
-        self.uid_forced = None
-        self.gid_forced = None
-        self.umask = 0
-        self.archive_root_dir = {}  # archive ID --> directory name
-
-        # Cache for file handles
-        self.handles = {}
+        self.vfs = None  # created by mount(), once the mount options are known
+        self.handles = {}  # file handle -> node
         self.handle_count = 0
 
-        # Cache for chunks (moved from ItemCache)
-        self.chunks_cache = LRUCache(capacity=10)
-
-    def _create_node(self, item=None, parent=None):
-        self.current_ino += 1
-        if item is not None:
-            self.set_inode(self.current_ino, item)
-        return DirEntry(self.current_ino, parent)
-
-    def get_inode(self, ino):
-        packed = self.inodes.get(ino)
-        if packed is None:
-            return None
-        return Item(internal_dict=msgpack.unpackb(packed))
-
-    def set_inode(self, ino, item):
-        if item is None:
-            self.inodes.pop(ino, None)
-        else:
-            # Remove path from the item dict before packing to save memory.
-            # The path is already encoded in the DirEntry tree structure.
-            item_dict = item.as_dict()
-            item_dict.pop("path", None)
-            self.inodes[ino] = msgpack.packb(item_dict)
-
-    def _create_filesystem(self):
-        self.set_inode(self.root.ino, self.default_dir)
-        self.versions_index = FuseVersionsIndex()
-
-        if getattr(self._args, "name", None):
-            archives = [self._manifest.archives.get(self._args.name)]
-        else:
-            archives = self._manifest.archives.list_considering(self._args)
-
-        name_counter = Counter(a.name for a in archives)
-        duplicate_names = {a.name for a in archives if name_counter[a.name] > 1}
-
-        for archive in archives:
-            name = f"{archive.name}"
-            if name in duplicate_names:
-                name += f"-{bin_to_hex(archive.id):.8}"
-            self.archive_root_dir[archive.id] = name
-
-        for archive in archives:
-            if self.versions:
-                self._process_archive(archive.id)
-            else:
-                # Create placeholder for archive
-                name = self.archive_root_dir[archive.id]
-                name_bytes = os.fsencode(name)
-
-                archive_node = self._create_node(parent=self.root)
-                # Create a directory item for the archive
-                item = Item(internal_dict=self.default_dir.as_dict())
-                item.mtime = int(archive.ts.timestamp() * 1e9)
-                self.set_inode(archive_node.ino, item)
-
-                self.root.add_child(name_bytes, archive_node)
-                self.pending_archives[archive_node] = archive
-
-    def check_pending_archive(self, node):
-        archive_info = self.pending_archives.pop(node, None)
-        if archive_info is not None:
-            with self._repo_lock:
-                self._process_archive(archive_info.id, node)
-
-    def _iter_archive_items(self, archive_item_ids, filter=None):
-        unpacker = msgpack.Unpacker()
-        for id, cdata in zip(archive_item_ids, self.repository.get_many(archive_item_ids)):
-            _, data = self.repo_objs.parse(id, cdata, ro_type=ROBJ_DONTCARE)
-            unpacker.feed(data)
-            for item in unpacker:
-                item = Item(internal_dict=item)
-                if filter and not filter(item):
-                    continue
-                yield item
-
-    def _process_archive(self, archive_id, root_node=None):
-        if root_node is None:
-            root_node = self.root
-
-        self.file_versions = {}  # for versions mode: original path -> version
-
-        archive = Archive(self._manifest, archive_id)
-        strip_components = self._args.strip_components
-        # omitting args.pattern_roots here, restricting to paths only by cli args.paths:
-        matcher = build_matcher(self._args.patterns, self._args.paths)
-        hlm = HardLinkManager(id_type=bytes, info_type=str)
-
-        filter = build_filter(matcher, strip_components)
-
-        for item in self._iter_archive_items(archive.metadata.items, filter=filter):
-            if strip_components:
-                item.path = os.sep.join(item.path.split(os.sep)[strip_components:])
-
-            path = os.fsencode(item.path)
-            segments = path.split(b"/")
-            is_dir = stat.S_ISDIR(item.mode)
-
-            # For versions mode, handle files differently
-            if self.versions and not is_dir:
-                self._process_leaf_versioned(segments, item, root_node, hlm)
-            else:
-                # Original non-versions logic
-                node = root_node
-                # Traverse/Create directories
-                for segment in segments[:-1]:
-                    if not node.has_child(segment):
-                        new_node = self._create_node(parent=node)
-                        # We might need a default directory item if it's an implicit directory
-                        self.set_inode(new_node.ino, Item(internal_dict=self.default_dir.as_dict()))
-                        node.add_child(segment, new_node)
-                    node = node.get_child(segment)
-
-                # Leaf (file or explicit directory)
-                leaf_name = segments[-1]
-                if node.has_child(leaf_name):
-                    # Already exists (e.g. implicit dir became explicit)
-                    child = node.get_child(leaf_name)
-                    self.set_inode(child.ino, item)  # Update item
-                    node = child
-                else:
-                    new_node = self._create_node(item, parent=node)
-                    node.add_child(leaf_name, new_node)
-                    node = new_node
-
-                # Handle hardlinks (non-versions mode)
-                if "hlid" in item:
-                    link_target = hlm.retrieve(id=item.hlid, default=None)
-                    if link_target is not None:
-                        target_path = os.fsencode(link_target)
-                        target_node = self._find_node_from_root(root_node, target_path)
-                        if target_node:
-                            # Reuse ino and item from target
-                            node.ino = target_node.ino
-                            # node.item = target_node.item  # implicitly shared via ID
-                            item = self.get_inode(node.ino)
-                            if "nlink" not in item:
-                                item.nlink = 1
-                            item.nlink += 1
-                            self.set_inode(node.ino, item)
-                        else:
-                            logger.warning("Hardlink target not found: %s", link_target)
-                    else:
-                        hlm.remember(id=item.hlid, info=item.path)
-
-    def _process_leaf_versioned(self, segments, item, root_node, hlm):
-        """Process a file leaf node in versions mode"""
-        path = b"/".join(segments)
-        original_path = item.path
-
-        # Handle hardlinks in versions mode - check if we've seen this hardlink before
-        is_hardlink = "hlid" in item
-        link_target = None
-        if is_hardlink:
-            link_target = hlm.retrieve(id=item.hlid, default=None)
-            if link_target is None:
-                # First occurrence of this hardlink
-                hlm.remember(id=item.hlid, info=original_path)
-
-        # Calculate version for this file
-        # If it's a hardlink to a previous file, use that version
-        if is_hardlink and link_target is not None:
-            link_target_enc = os.fsencode(link_target)
-            version = self.file_versions.get(link_target_enc)
-        else:
-            version = self._file_version(item, path)
-
-        # Store version for this path
-        if version is not None:
-            self.file_versions[path] = version
-
-        # Navigate to parent directory
-        node = root_node
-        for segment in segments[:-1]:
-            if not node.has_child(segment):
-                new_node = self._create_node(parent=node)
-                self.set_inode(new_node.ino, Item(internal_dict=self.default_dir.as_dict()))
-                node.add_child(segment, new_node)
-            node = node.get_child(segment)
-
-        # Create intermediate directory with the filename
-        leaf_name = segments[-1]
-        if not node.has_child(leaf_name):
-            intermediate_node = self._create_node(parent=node)
-            self.set_inode(intermediate_node.ino, Item(internal_dict=self.default_dir.as_dict()))
-            node.add_child(leaf_name, intermediate_node)
-        else:
-            intermediate_node = node.get_child(leaf_name)
-
-        # Create versioned filename
-        if version is not None:
-            versioned_name = self._make_versioned_name(leaf_name, version)
-
-            # If this is a hardlink to a previous file, reuse that node
-            if is_hardlink and link_target is not None:
-                link_target_enc = os.fsencode(link_target)
-                link_segments = link_target_enc.split(b"/")
-                link_version = self.file_versions.get(link_target_enc)
-                if link_version is not None:
-                    # Navigate to the link target
-                    target_node = root_node
-                    for seg in link_segments[:-1]:
-                        if target_node.has_child(seg):
-                            target_node = target_node.get_child(seg)
-                        else:
-                            break
-                    else:
-                        # Get intermediate dir
-                        link_leaf = link_segments[-1]
-                        if target_node.has_child(link_leaf):
-                            target_intermediate = target_node.get_child(link_leaf)
-                            target_versioned = self._make_versioned_name(link_leaf, link_version)
-                            if target_intermediate.has_child(target_versioned):
-                                original_node = target_intermediate.get_child(target_versioned)
-                                # Create new node but reuse the ino and item from original
-                                item = self.get_inode(original_node.ino)
-                                file_node = self._create_node(item, parent=intermediate_node)
-                                file_node.ino = original_node.ino
-                                # Update nlink count
-                                item = self.get_inode(file_node.ino)
-                                if "nlink" not in item:
-                                    item.nlink = 1
-                                item.nlink += 1
-                                self.set_inode(file_node.ino, item)
-                                intermediate_node.add_child(versioned_name, file_node)
-                                return
-
-            # Not a hardlink or first occurrence - create new node
-            file_node = self._create_node(item, parent=intermediate_node)
-            intermediate_node.add_child(versioned_name, file_node)
-
-    def _file_version(self, item, path):
-        """Calculate version number for a file based on its contents"""
-        if "chunks" not in item:
-            return None
-
-        # note: using sha256 here because nowadays it is often hw accelerated.
-        #       shortening the hashes to 16 bytes to save some memory.
-        file_id = hashlib.sha256(path).digest()[:16]
-        current_version, previous_id = self.versions_index.get(file_id, (0, None))
-
-        contents_id = hashlib.sha256(b"".join(chunk_id for chunk_id, _ in item.chunks)).digest()[:16]
-
-        if contents_id != previous_id:
-            current_version += 1
-            self.versions_index[file_id] = current_version, contents_id
-
-        return current_version
-
-    def _make_versioned_name(self, name, version):
-        """Generate versioned filename like 'file.00001.txt'"""
-        # keep original extension at end to avoid confusing tools
-        name_str = name.decode("utf-8", "surrogateescape") if isinstance(name, bytes) else name
-        name_part, ext = os.path.splitext(name_str)
-        version_str = ".%05d" % version
-        versioned = name_part + version_str + ext
-        return versioned.encode("utf-8", "surrogateescape") if isinstance(name, bytes) else versioned
-
-    def _find_node_from_root(self, root, path):
-        if path == b"" or path == b".":
-            return root
-        segments = path.split(b"/")
-        node = root
-        for segment in segments:
-            child = node.get_child(segment)
-            if child is not None:
-                node = child
-            else:
-                return None
-        return node
-
-    def _find_node(self, path):
-        if isinstance(path, str):
-            path = os.fsencode(path)
-        if path == b"/" or path == b"":
-            return self.root
-        if path.startswith(b"/"):
-            path = path[1:]
-
-        segments = path.split(b"/")
-        node = self.root
-        for segment in segments:
-            if node in self.pending_archives:
-                self.check_pending_archive(node)
-            child = node.get_child(segment)
-            if child is not None:
-                node = child
-            else:
-                return None
-
-        if node in self.pending_archives:
-            self.check_pending_archive(node)
-
-        return node
-
-    def _get_handle(self, node):
-        self.handle_count += 1
-        self.handles[self.handle_count] = node
-        return self.handle_count
-
-    def _get_node_from_handle(self, fh):
-        return self.handles.get(fh)
-
-    def _make_stat_dict(self, node):
-        """Create a stat dictionary from a node."""
-        item = self.get_inode(node.ino)
-        st = {}
-        st["st_ino"] = node.ino
-        st["st_mode"] = item.mode & ~self.umask
-        st["st_nlink"] = item.get("nlink", 1)
-        if stat.S_ISDIR(st["st_mode"]):
-            st["st_nlink"] = max(st["st_nlink"], 2)
-        st["st_uid"], st["st_gid"] = get_item_uid_gid(
-            item,
-            numeric=self.numeric_ids,
-            uid_default=self.default_uid,
-            gid_default=self.default_gid,
-            uid_forced=self.uid_forced,
-            gid_forced=self.gid_forced,
-        )
-        st["st_rdev"] = item.get("rdev", 0)
-        st["st_size"] = item.get_size()
-        st["st_blocks"] = (st["st_size"] + BLOCK_SIZE - 1) // BLOCK_SIZE
-        if getattr(self, "use_ns", False):
-            st["st_mtime"] = item.mtime
-            st["st_atime"] = item.get("atime", item.mtime)
-            st["st_ctime"] = item.get("ctime", item.mtime)
-        else:
-            st["st_mtime"] = item.mtime / 1e9
-            st["st_atime"] = item.get("atime", item.mtime) / 1e9
-            st["st_ctime"] = item.get("ctime", item.mtime) / 1e9
-        return st
-
-
-class borgfs(hlfuse.Operations, FuseBackend):
-    """Export archive as a FUSE filesystem"""
-
-    use_ns = True
-
-    def __init__(self, manifest, args, repository):
-        hlfuse.Operations.__init__(self)
-        FuseBackend.__init__(self, manifest, args, repository)
-        data_cache_capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1))
-        logger.debug("mount data cache capacity: %d chunks", data_cache_capacity)
-        self.data_cache = LRUCache(capacity=data_cache_capacity)
-        self._last_pos = LRUCache(capacity=4)
-
-    def sig_info_handler(self, sig_no, stack):
-        # Simplified instrumentation
-        logger.debug("fuse: %d inodes", self.current_ino)
-
     def mount(self, mountpoint, mount_options, foreground=False, show_rc=False):
         """Mount filesystem on *mountpoint* with *mount_options*."""
-
-        def pop_option(options, key, present, not_present, wanted_type, int_base=0):
-            assert isinstance(options, list)  # we mutate this
-            for idx, option in enumerate(options):
-                if option == key:
-                    options.pop(idx)
-                    return present
-                if option.startswith(key + "="):
-                    options.pop(idx)
-                    value = option.split("=", 1)[1]
-                    if wanted_type is bool:
-                        v = value.lower()
-                        if v in ("y", "yes", "true", "1"):
-                            return True
-                        if v in ("n", "no", "false", "0"):
-                            return False
-                        raise ValueError("unsupported value in option: %s" % option)
-                    if wanted_type is int:
-                        try:
-                            return int(value, base=int_base)
-                        except ValueError:
-                            raise ValueError("unsupported value in option: %s" % option) from None
-                    try:
-                        return wanted_type(value)
-                    except ValueError:
-                        raise ValueError("unsupported value in option: %s" % option) from None
-            else:
-                return not_present
-
-        options = ["fsname=borgfs", "ro", "default_permissions"]
-        if mount_options:
-            options.extend(mount_options.split(","))
-        if is_darwin:
-            volname = pop_option(options, "volname", "", "", str)
-            volname = volname or f"{os.path.basename(mountpoint)} (borgfs)"
-            options.append(f"volname={volname}")
-        ignore_permissions = pop_option(options, "ignore_permissions", True, False, bool)
-        if ignore_permissions:
-            pop_option(options, "default_permissions", True, False, bool)
-        self.allow_damaged_files = pop_option(options, "allow_damaged_files", True, False, bool)
-        self.versions = pop_option(options, "versions", True, False, bool)
-        self.uid_forced = pop_option(options, "uid", None, None, int)
-        self.gid_forced = pop_option(options, "gid", None, None, int)
-        self.umask = pop_option(options, "umask", 0, 0, int, int_base=8)
-        dir_uid = self.uid_forced if self.uid_forced is not None else self.default_uid
-        dir_gid = self.gid_forced if self.gid_forced is not None else self.default_gid
-        dir_user = uid2user(dir_uid)
-        dir_group = gid2group(dir_gid)
-        if not isinstance(dir_user, str):
-            raise Error(
-                f"uid {dir_uid} can not be resolved to a username. "
-                f"Please check that the corresponding user exists or do not specify a uid mount option."
-            )
-        if not isinstance(dir_group, str):
-            raise Error(
-                f"gid {dir_gid} can not be resolved to a group name. "
-                f"Please check that the corresponding group exists or do not specify a gid mount option."
-            )
-        dir_mode = 0o40755 & ~self.umask
-        self.default_dir = Item(
-            mode=dir_mode, mtime=int(time.time() * 1e9), user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid
-        )
-        self._create_filesystem()
+        options, vfs_options = parse_mount_options(self._args, mountpoint, mount_options)
+        self.vfs = ArchiveVFS(self._manifest, self._args, self._repository, lock=self._repo_lock, options=vfs_options)
+        self.vfs.create_filesystem()
 
         # hlfuse.FUSE will block if foreground=True, otherwise it returns immediately
         if not foreground:
             # Background mode: daemonize first, then start FUSE (blocking)
             with daemonizing(show_rc=show_rc) as (old_id, new_id):
                 logger.debug("fuse: mount repo, going to background: migrating lock.")
-                self.repository.migrate_lock(old_id, new_id)
+                self._repository.migrate_lock(old_id, new_id)
 
         # keep the repository lock of an idle mount alive, so it is not killed as stale (see #9872).
         # started here (after a possible daemonizing fork, as threads do not survive fork()).
-        lock_refreshing_thread = LockRefresher(self.repository.info, sleep_interval=60, lock=self._repo_lock)
+        lock_refreshing_thread = LockRefresher(self._repository.info, sleep_interval=60, lock=self._repo_lock)
         lock_refreshing_thread.start()
         try:
             # Run the FUSE main loop in foreground (we might be daemonized already or not)
@@ -555,183 +70,115 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
         finally:
             lock_refreshing_thread.terminate()
 
-    def statfs(self, path):
-        debug_log(f"statfs(path={path!r})")
-        stat_ = {}
-        stat_["f_bsize"] = BLOCK_SIZE
-        stat_["f_frsize"] = BLOCK_SIZE
-        stat_["f_blocks"] = 0
-        stat_["f_bfree"] = 0
-        stat_["f_bavail"] = 0
-        stat_["f_files"] = 0
-        stat_["f_ffree"] = 0
-        stat_["f_favail"] = 0
-        stat_["f_namemax"] = 255
-        debug_log(f"statfs -> {stat_}")
-        return stat_
+    def sig_info_handler(self, sig_no, stack):
+        self.vfs.log_stats()
 
-    def getattr(self, path, fh=None):
-        debug_log(f"getattr(path={path!r}, fh={fh})")
-        if fh is not None:
-            # use file handle if available to avoid path lookup
-            node = self._get_node_from_handle(fh)
-            if node is None:
-                raise hlfuse.FuseOSError(errno.EBADF)
+    # -- helpers ---------------------------------------------------------------
+
+    def _find_node(self, path):
+        """Return the node at *path*; raises ENOENT if there is none."""
+        segments = [segment for segment in path.split("/") if segment]
+        try:
+            _, node = self.vfs.resolve(segments)
+        except KeyError:
+            raise hlfuse.FuseOSError(errno.ENOENT) from None
+        return node
+
+    def _node_from_handle(self, fh):
+        node = self.handles.get(fh)
+        if node is None:
+            raise hlfuse.FuseOSError(errno.EBADF)
+        return node
+
+    def _stat(self, node):
+        """Build the stat dict of *node*."""
+        attrs = self.vfs.attrs(node.ino)
+        st = {
+            "st_ino": attrs.ino,
+            "st_mode": attrs.mode,
+            "st_nlink": attrs.nlink,
+            "st_uid": attrs.uid,
+            "st_gid": attrs.gid,
+            "st_rdev": attrs.rdev,
+            "st_size": attrs.size,
+            "st_blocks": (attrs.size + BLOCK_SIZE - 1) // BLOCK_SIZE,
+        }
+        if self.use_ns:
+            st["st_mtime"] = attrs.mtime_ns
+            st["st_atime"] = attrs.atime_ns
+            st["st_ctime"] = attrs.ctime_ns
         else:
-            node = self._find_node(path)
-            if node is None:
-                raise hlfuse.FuseOSError(errno.ENOENT)
-        st = self._make_stat_dict(node)
-        debug_log(f"getattr -> {st}")
+            st["st_mtime"] = attrs.mtime_ns / 1e9
+            st["st_atime"] = attrs.atime_ns / 1e9
+            st["st_ctime"] = attrs.ctime_ns / 1e9
         return st
 
+    # -- filesystem operations -------------------------------------------------
+
+    def statfs(self, path):
+        return {
+            "f_bsize": BLOCK_SIZE,
+            "f_frsize": BLOCK_SIZE,
+            "f_blocks": 0,
+            "f_bfree": 0,
+            "f_bavail": 0,
+            "f_files": 0,
+            "f_ffree": 0,
+            "f_favail": 0,
+            "f_namemax": 255,  # == NAME_MAX (depends on archive source OS / FS)
+        }
+
+    def getattr(self, path, fh=None):
+        # use the file handle if we have one, to avoid the path lookup
+        node = self._node_from_handle(fh) if fh is not None else self._find_node(path)
+        return self._stat(node)
+
     def listxattr(self, path):
-        debug_log(f"listxattr(path={path!r})")
         node = self._find_node(path)
-        if node is None:
-            raise hlfuse.FuseOSError(errno.ENOENT)
-        item = self.get_inode(node.ino)
-        result = [k.decode("utf-8", "surrogateescape") for k in item.get("xattrs", {}).keys()]
-        # expose the archived POSIX ACLs, so e.g. getfacl or tools copying from the mount can read them.
-        result.extend(xattr_name for xattr_name, attr in ACL_XATTRS.items() if attr in item)
-        debug_log(f"listxattr -> {result}")
-        return result
+        return [name.decode("utf-8", "surrogateescape") for name in self.vfs.listxattr(node.ino)]
 
     def getxattr(self, path, name, position=0):
-        debug_log(f"getxattr(path={path!r}, name={name!r}, position={position})")
         node = self._find_node(path)
-        if node is None:
-            raise hlfuse.FuseOSError(errno.ENOENT)
-        item = self.get_inode(node.ino)
-        name_str = name if isinstance(name, str) else name.decode("utf-8", "surrogateescape")
-        if name_str in ACL_XATTRS:
-            acl = item.get(ACL_XATTRS[name_str])
-            if acl is None:
-                debug_log("getxattr -> ENOATTR")
-                raise hlfuse.FuseOSError(ENOATTR)
-            try:
-                result = acl_text_to_xattr(acl, numeric_ids=self.numeric_ids)
-            except ValueError:
-                logger.warning(f"mount: could not convert ACL of {path!r} to the xattr representation")
-                raise hlfuse.FuseOSError(errno.EIO) from None
-            debug_log(f"getxattr -> {len(result)} bytes")
-            return result
+        if isinstance(name, str):
+            name = name.encode("utf-8", "surrogateescape")
         try:
-            if isinstance(name, str):
-                name = name.encode("utf-8", "surrogateescape")
-            result = item.get("xattrs", {})[name] or b""
-            debug_log(f"getxattr -> {len(result)} bytes")
-            return result
+            return self.vfs.getxattr(node.ino, name)
         except KeyError:
-            debug_log("getxattr -> ENOATTR")
             raise hlfuse.FuseOSError(ENOATTR) from None
+        except ValueError:
+            logger.warning(f"mount: could not convert ACL of {path!r} to the xattr representation")
+            raise hlfuse.FuseOSError(errno.EIO) from None
 
     def open(self, path, fi):
-        debug_log(f"open(path={path!r}, fi={fi})")
         node = self._find_node(path)
-        if node is None:
-            raise hlfuse.FuseOSError(errno.ENOENT)
-        fh = self._get_handle(node)
-        fi.fh = fh
-        debug_log(f"open -> fh={fh}")
+        self.handle_count += 1
+        self.handles[self.handle_count] = node
+        fi.fh = self.handle_count
         return 0
 
     def release(self, path, fi):
-        debug_log(f"release(path={path!r}, fh={fi.fh})")
         self.handles.pop(fi.fh, None)
-        self._last_pos.pop(fi.fh, None)
+        self.vfs.reader.forget(fi.fh)
         return 0
 
     def create(self, path, mode, fi=None):
-        debug_log(f"create(path={path!r}, mode={mode}, fi={fi}) -> EROFS")
         raise hlfuse.FuseOSError(errno.EROFS)
 
     def read(self, path, size, offset, fi):
-        fh = fi.fh
-        debug_log(f"read(path={path!r}, size={size}, offset={offset}, fh={fh})")
-        node = self._get_node_from_handle(fh)
-        if node is None:
-            raise hlfuse.FuseOSError(errno.EBADF)
-
-        item = self.get_inode(node.ino)
-        parts = []
-
-        # optimize for linear reads:
-        chunk_no, chunk_offset = self._last_pos.get(fh, (0, 0))
-        if chunk_offset > offset:
-            chunk_no, chunk_offset = (0, 0)
-
-        offset -= chunk_offset
-        chunks = item.chunks
-
-        for idx in range(chunk_no, len(chunks)):
-            id, s = chunks[idx]
-            if s < offset:
-                offset -= s
-                chunk_offset += s
-                chunk_no += 1
-                continue
-            n = min(size, s - offset)
-            if id in self.data_cache:
-                data = self.data_cache[id]
-                if offset + n == len(data):
-                    del self.data_cache[id]
-            else:
-                try:
-                    # Direct repository access
-                    with self._repo_lock:
-                        cdata = self.repository.get(id)
-                except Repository.ObjectNotFound:
-                    if self.allow_damaged_files:
-                        data = zeros[:s]
-                        assert len(data) == s
-                    else:
-                        raise hlfuse.FuseOSError(errno.EIO) from None
-                else:
-                    _, data = self.repo_objs.parse(id, cdata, ro_type=ROBJ_FILE_STREAM)
-                if offset + n < len(data):
-                    self.data_cache[id] = data
-            parts.append(data[offset : offset + n])
-            offset = 0
-            size -= n
-            if not size:
-                if fh in self._last_pos:
-                    self._last_pos.replace(fh, (chunk_no, chunk_offset))
-                else:
-                    self._last_pos[fh] = (chunk_no, chunk_offset)
-                break
-        result = b"".join(parts)
-        debug_log(f"read -> {len(result)} bytes")
-        return result
+        node = self._node_from_handle(fi.fh)
+        try:
+            return self.vfs.read(node.ino, offset, size, pos_key=fi.fh)
+        except ChunkMissing:
+            raise hlfuse.FuseOSError(errno.EIO) from None
 
     def readdir(self, path, fh=None):
-        debug_log(f"readdir(path={path!r}, fh={fh})")
         node = self._find_node(path)
-        if node is None:
-            raise hlfuse.FuseOSError(errno.ENOENT)
-
-        offset = 0
-        offset += 0  # += 1
-        debug_log(f"readdir yielding . {offset}")
-        yield (".", self._make_stat_dict(node), offset)
-        offset += 0  # += 1
-        debug_log(f"readdir yielding .. {offset}")
-        parent = node.parent if node.parent else node
-        yield ("..", self._make_stat_dict(parent), offset)
-
-        for name, child_node in node.iter_children():
-            name_str = name.decode("utf-8", "surrogateescape")
-            st = self._make_stat_dict(child_node)
-            offset += 0  # += 1
-            debug_log(f"readdir yielding {name_str} {offset} {st}")
-            yield (name_str, st, offset)
+        # offset 0 for all entries: we always return the full directory at once.
+        yield (".", self._stat(node), 0)
+        yield ("..", self._stat(node.parent if node.parent else node), 0)
+        for name, child in self.vfs.children(node):
+            yield (name, self._stat(child), 0)
 
     def readlink(self, path):
-        debug_log(f"readlink(path={path!r})")
         node = self._find_node(path)
-        if node is None:
-            raise hlfuse.FuseOSError(errno.ENOENT)
-        item = self.get_inode(node.ino)
-        result = item.target
-        debug_log(f"readlink -> {result!r}")
-        return result
+        return self.vfs.readlink(node.ino)
diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py
index c9fab37834..79d4d04934 100644
--- a/src/borg/testsuite/archiver/webdav_cmd_test.py
+++ b/src/borg/testsuite/archiver/webdav_cmd_test.py
@@ -11,7 +11,6 @@
 import tarfile
 import threading
 import time
-import unicodedata
 import urllib.error
 import urllib.request
 import xml.etree.ElementTree as ET
@@ -26,7 +25,7 @@
 from ...manifest import Manifest
 from ...platform import is_win32
 from ...repository import Repository
-from ...webdav import make_server, lookup_child, Node
+from ...webdav import make_server
 from .. import are_symlinks_supported, are_hardlinks_supported
 from . import RK_ENCRYPTION, cmd, create_regular_file, generate_archiver_tests
 
@@ -468,18 +467,23 @@ def test_webdav_data_cache(archivers, request, monkeypatch):
     with repository:
         manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
         server = make_server(manifest, args, port=0)
-        assert server.data_cache._capacity == 8  # the env var is honored
+        data_cache = server.RequestHandlerClass.vfs.reader.data_cache
+        assert data_cache._capacity == 8  # the env var is honored
         thread = threading.Thread(target=server.serve_forever, daemon=True)
         thread.start()
         try:
             url = f"http://127.0.0.1:{server.server_address[1]}/test/input/big"
-            assert len(server.data_cache) == 0
-            status, _, body = get(url)  # first read populates the cache
+            assert len(data_cache) == 0
+            status, _, body = get(url)  # a full download reads each chunk completely ...
             assert status == 200 and body == big
-            assert len(server.data_cache) > 0  # the multi-chunk file cached some chunks
-            # a second (ranged) read hits the cache and must return identical bytes
+            assert len(data_cache) == 0  # ... so it does not fill (pollute) the cache
+            # a range request only reads part of a chunk, so that chunk is worth caching
             status, _, body = get(url, headers={"Range": "bytes=0-99"})
             assert status == 206 and body == big[:100]
+            assert len(data_cache) > 0
+            # the next range request in the same chunk is served from the cache
+            status, _, body = get(url, headers={"Range": "bytes=100-199"})
+            assert status == 206 and body == big[100:200]
         finally:
             server.shutdown()
             server.server_close()
@@ -522,36 +526,6 @@ def test_webdav_unicode_normalization(archivers, request):
         assert exc_info.value.code == 404
 
 
-def test_webdav_lookup_child():
-    # exact matches always win, and names that are ambiguous after normalization are only
-    # reachable by their exact spelling, so we never serve a different file than requested.
-    nfc_name = "grüße.txt"  # composed
-    nfd_name = unicodedata.normalize("NFD", nfc_name)  # decomposed
-    assert nfc_name != nfd_name
-
-    only_nfc = Node(0o40755, children={nfc_name: Node(0o100644)})
-    # the stored (composed) name is found by both spellings
-    assert lookup_child(only_nfc, nfc_name)[0] == nfc_name
-    assert lookup_child(only_nfc, nfd_name)[0] == nfc_name
-
-    only_nfd = Node(0o40755, children={nfd_name: Node(0o100644)})
-    assert lookup_child(only_nfd, nfd_name)[0] == nfd_name
-    assert lookup_child(only_nfd, nfc_name)[0] == nfd_name  # the other way round, too
-
-    # an archive may contain both spellings (they are different names on e.g. Linux):
-    # each one resolves to itself, exactly.
-    both = Node(0o40755, children={nfc_name: Node(0o100644), nfd_name: Node(0o100644)})
-    assert lookup_child(both, nfc_name)[0] == nfc_name
-    assert lookup_child(both, nfd_name)[0] == nfd_name
-
-    # a name with non-UTF-8 bytes (surrogate escapes) must not break normalization
-    weird = b"bad\xff.txt".decode("utf-8", "surrogateescape")
-    assert lookup_child(Node(0o40755, children={weird: Node(0o100644)}), weird)[0] == weird
-
-    with pytest.raises(KeyError):
-        lookup_child(only_nfc, "no-such-file.txt")
-
-
 def test_webdav_file_without_chunks(archivers, request):
     # An anomalous item - size > 0 but no chunks list (e.g. corrupted metadata) - must not
     # leave the client hanging after an advertised Content-Length: the server aborts the
@@ -566,9 +540,12 @@ def test_webdav_file_without_chunks(archivers, request):
         manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
         server = make_server(manifest, args, port=0)
         # corrupt the in-memory tree: pretend file1 is non-empty but has no chunks
-        node, _, _ = server.RequestHandlerClass.vfs.resolve(["test", "input", "file1"])
-        node.size = 5
-        node.chunks = None
+        vfs = server.RequestHandlerClass.vfs
+        _, node = vfs.resolve(["test", "input", "file1"])
+        item = vfs.get_item(node.ino)
+        item.size = 5
+        del item.chunks
+        vfs._set_item(node.ino, item)
         thread = threading.Thread(target=server.serve_forever, daemon=True)
         thread.start()
         try:
diff --git a/src/borg/testsuite/fuse_test.py b/src/borg/testsuite/fuse_test.py
index dda2b24c40..e18440dc24 100644
--- a/src/borg/testsuite/fuse_test.py
+++ b/src/borg/testsuite/fuse_test.py
@@ -1,38 +1,35 @@
-"""Tests for the FUSE filesystem implementations that do not need an actual mount.
+"""Tests for the FUSE adapters that do not need an actual mount.
 
-The ACL xattrs (see ACL_XATTRS) are emulated by the FUSE implementations, so we can
-test them by directly calling the relevant methods with a fake item. Unlike the tests
-in archiver/mount_cmds_test.py, this also works in environments where the kernel does
-not offer ACL xattrs on FUSE mounts (e.g. mounts made inside a user namespace).
+The archive VFS (see vfs_test.py) reports a missing xattr as a KeyError and an ACL it
+can not convert as a ValueError - both FUSE adapters have to turn these into the right
+errno, which is what we check here without mounting anything.
 """
 
 import errno
 
 import pytest
 
-from ..helpers import StableDict
-from ..item import Item
-from ..platform import acl_text_to_xattr
 from . import has_llfuse, has_pyfuse3, has_mfusepy, ENOATTR
-from .platform.platform_test import skipif_not_linux
-
-# the ACL xattrs are only emulated on Linux:
-pytestmark = skipif_not_linux
 
 skipif_no_llfuse_api = pytest.mark.skipif(not (has_llfuse or has_pyfuse3), reason="llfuse/pyfuse3 not available")
 skipif_no_mfusepy = pytest.mark.skipif(not has_mfusepy, reason="mfusepy not available")
 
-ACCESS_ACL = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:0\ngroup:root:rw-:0\n"
-DEFAULT_ACL = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:r--:0\ngroup:root:r--:0\n"
-BROKEN_ACL = b"flubber::rw-\n"  # can not be converted to the binary xattr representation
+XATTRS = [b"user.foo", b"system.posix_acl_access"]
+
+
+class FakeVFS:
+    """Minimal stand-in for the archive VFS: answers the xattr calls, however we want."""
 
+    def __init__(self, exception=None):
+        self.exception = exception
 
-def make_item(*, acls=True, acl_access=ACCESS_ACL):
-    item = Item(path="file", mode=0o100666, mtime=0, xattrs=StableDict({b"user.foo": b"bar"}))
-    if acls:
-        item.acl_access = acl_access
-        item.acl_default = DEFAULT_ACL
-    return item
+    def listxattr(self, ino):
+        return list(XATTRS)
+
+    def getxattr(self, ino, name):
+        if self.exception is not None:
+            raise self.exception
+        return b"bar"
 
 
 def unwrap(method):
@@ -40,21 +37,18 @@ def unwrap(method):
     return getattr(method, "__wrapped__", method)
 
 
-def llfuse_ops(item, numeric_ids=False):
-    """Minimal stand-in for FuseOperations, providing what listxattr/getxattr need."""
+def llfuse_ops(exception=None):
     from ..fuse import FuseOperations
 
     class Operations:
         pass
 
     ops = Operations()
-    ops.numeric_ids = numeric_ids
-    ops.get_item = lambda inode: item
+    ops.vfs = FakeVFS(exception)
     return ops, unwrap(FuseOperations.listxattr), unwrap(FuseOperations.getxattr)
 
 
-def mfusepy_ops(item, numeric_ids=False):
-    """Minimal stand-in for borgfs, providing what listxattr/getxattr need."""
+def mfusepy_ops(exception=None):
     from ..hlfuse import borgfs
 
     class Node:
@@ -64,91 +58,71 @@ class Operations:
         pass
 
     ops = Operations()
-    ops.numeric_ids = numeric_ids
+    ops.vfs = FakeVFS(exception)
     ops._find_node = lambda path: Node()
-    ops.get_inode = lambda ino: item
     return ops, borgfs.listxattr, borgfs.getxattr
 
 
 @skipif_no_llfuse_api
-def test_llfuse_listxattr_acls():
-    ops, listxattr, _ = llfuse_ops(make_item())
-    assert sorted(listxattr(ops, 1)) == [b"system.posix_acl_access", b"system.posix_acl_default", b"user.foo"]
+def test_llfuse_listxattr():
+    ops, listxattr, _ = llfuse_ops()
+    assert listxattr(ops, 1) == XATTRS  # the low-level API uses bytes names
 
 
 @skipif_no_llfuse_api
-def test_llfuse_listxattr_no_acls():
-    ops, listxattr, _ = llfuse_ops(make_item(acls=False))
-    assert list(listxattr(ops, 1)) == [b"user.foo"]
+def test_llfuse_getxattr():
+    ops, _, getxattr = llfuse_ops()
+    assert getxattr(ops, 1, b"user.foo") == b"bar"
 
 
 @skipif_no_llfuse_api
-@pytest.mark.parametrize("numeric_ids", [False, True])
-def test_llfuse_getxattr_acls(numeric_ids):
-    ops, _, getxattr = llfuse_ops(make_item(), numeric_ids=numeric_ids)
-    assert getxattr(ops, 1, b"system.posix_acl_access") == acl_text_to_xattr(ACCESS_ACL, numeric_ids=numeric_ids)
-    assert getxattr(ops, 1, b"system.posix_acl_default") == acl_text_to_xattr(DEFAULT_ACL, numeric_ids=numeric_ids)
-    assert getxattr(ops, 1, b"user.foo") == b"bar"  # "normal" xattrs still work
-
-
-@skipif_no_llfuse_api
-def test_llfuse_getxattr_acl_missing():
+def test_llfuse_getxattr_missing():
     from ..fuse_impl import llfuse
 
-    ops, _, getxattr = llfuse_ops(make_item(acls=False))
+    ops, _, getxattr = llfuse_ops(KeyError(b"user.foo"))
     with pytest.raises(llfuse.FUSEError) as excinfo:
-        getxattr(ops, 1, b"system.posix_acl_access")
+        getxattr(ops, 1, b"user.foo")
     assert excinfo.value.errno == ENOATTR
 
 
 @skipif_no_llfuse_api
-def test_llfuse_getxattr_acl_broken():
+def test_llfuse_getxattr_broken_acl():
     from ..fuse_impl import llfuse
 
-    ops, _, getxattr = llfuse_ops(make_item(acl_access=BROKEN_ACL))
+    ops, _, getxattr = llfuse_ops(ValueError("can not convert this ACL"))
     with pytest.raises(llfuse.FUSEError) as excinfo:
         getxattr(ops, 1, b"system.posix_acl_access")
     assert excinfo.value.errno == errno.EIO
 
 
 @skipif_no_mfusepy
-def test_mfusepy_listxattr_acls():
-    ops, listxattr, _ = mfusepy_ops(make_item())
-    assert sorted(listxattr(ops, "/file")) == ["system.posix_acl_access", "system.posix_acl_default", "user.foo"]
-
-
-@skipif_no_mfusepy
-def test_mfusepy_listxattr_no_acls():
-    ops, listxattr, _ = mfusepy_ops(make_item(acls=False))
-    assert list(listxattr(ops, "/file")) == ["user.foo"]
+def test_mfusepy_listxattr():
+    ops, listxattr, _ = mfusepy_ops()
+    # the high-level API uses str names
+    assert listxattr(ops, "/file") == ["user.foo", "system.posix_acl_access"]
 
 
 @skipif_no_mfusepy
-@pytest.mark.parametrize("numeric_ids", [False, True])
-def test_mfusepy_getxattr_acls(numeric_ids):
-    ops, _, getxattr = mfusepy_ops(make_item(), numeric_ids=numeric_ids)
-    access = getxattr(ops, "/file", "system.posix_acl_access")
-    assert access == acl_text_to_xattr(ACCESS_ACL, numeric_ids=numeric_ids)
-    default = getxattr(ops, "/file", "system.posix_acl_default")
-    assert default == acl_text_to_xattr(DEFAULT_ACL, numeric_ids=numeric_ids)
-    assert getxattr(ops, "/file", "user.foo") == b"bar"  # "normal" xattrs still work
+def test_mfusepy_getxattr():
+    ops, _, getxattr = mfusepy_ops()
+    assert getxattr(ops, "/file", "user.foo") == b"bar"
 
 
 @skipif_no_mfusepy
-def test_mfusepy_getxattr_acl_missing():
+def test_mfusepy_getxattr_missing():
     from ..fuse_impl import hlfuse
 
-    ops, _, getxattr = mfusepy_ops(make_item(acls=False))
+    ops, _, getxattr = mfusepy_ops(KeyError(b"user.foo"))
     with pytest.raises(hlfuse.FuseOSError) as excinfo:
-        getxattr(ops, "/file", "system.posix_acl_default")
+        getxattr(ops, "/file", "user.foo")
     assert excinfo.value.errno == ENOATTR
 
 
 @skipif_no_mfusepy
-def test_mfusepy_getxattr_acl_broken():
+def test_mfusepy_getxattr_broken_acl():
     from ..fuse_impl import hlfuse
 
-    ops, _, getxattr = mfusepy_ops(make_item(acl_access=BROKEN_ACL))
+    ops, _, getxattr = mfusepy_ops(ValueError("can not convert this ACL"))
     with pytest.raises(hlfuse.FuseOSError) as excinfo:
         getxattr(ops, "/file", "system.posix_acl_access")
     assert excinfo.value.errno == errno.EIO
diff --git a/src/borg/testsuite/vfs_test.py b/src/borg/testsuite/vfs_test.py
new file mode 100644
index 0000000000..edef69485f
--- /dev/null
+++ b/src/borg/testsuite/vfs_test.py
@@ -0,0 +1,114 @@
+"""Tests for the archive VFS core (vfs.py) that do not need a repository."""
+
+import unicodedata
+
+import pytest
+
+from ..helpers import StableDict
+from ..item import Item
+from ..platform import acl_text_to_xattr
+from ..vfs import VFSNode, item_getxattr, item_listxattr, lookup_child, versioned_name
+from .platform.platform_test import skipif_not_linux
+
+ACCESS_ACL = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:0\ngroup:root:rw-:0\n"
+DEFAULT_ACL = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:r--:0\ngroup:root:r--:0\n"
+BROKEN_ACL = b"flubber::rw-\n"  # can not be converted to the binary xattr representation
+
+
+def make_item(*, acls=True, acl_access=ACCESS_ACL):
+    item = Item(path="file", mode=0o100666, mtime=0, xattrs=StableDict({b"user.foo": b"bar"}))
+    if acls:
+        item.acl_access = acl_access
+        item.acl_default = DEFAULT_ACL
+    return item
+
+
+def make_dir(names):
+    """Build a directory node with a leaf child for each name."""
+    node = VFSNode(1, is_dir=True)
+    for ino, name in enumerate(names, start=2):
+        node.children[name] = VFSNode(ino, node)
+    return node
+
+
+def test_versioned_name():
+    assert versioned_name("file.txt", 1) == "file.00001.txt"  # the extension stays at the end
+    assert versioned_name("file", 42) == "file.00042"
+
+
+def test_lookup_child():
+    # exact matches always win, and names that are ambiguous after normalization are only
+    # reachable by their exact spelling, so we never serve a different file than requested.
+    nfc_name = "grüße.txt"  # composed
+    nfd_name = unicodedata.normalize("NFD", nfc_name)  # decomposed
+    assert nfc_name != nfd_name
+
+    only_nfc = make_dir([nfc_name])
+    # the stored (composed) name is found by both spellings
+    assert lookup_child(only_nfc, nfc_name)[0] == nfc_name
+    assert lookup_child(only_nfc, nfd_name)[0] == nfc_name
+
+    only_nfd = make_dir([nfd_name])
+    assert lookup_child(only_nfd, nfd_name)[0] == nfd_name
+    assert lookup_child(only_nfd, nfc_name)[0] == nfd_name  # the other way round, too
+
+    # an archive may contain both spellings (they are different names on e.g. Linux):
+    # each one resolves to itself, exactly.
+    both = make_dir([nfc_name, nfd_name])
+    assert lookup_child(both, nfc_name)[0] == nfc_name
+    assert lookup_child(both, nfd_name)[0] == nfd_name
+
+    # a name with non-UTF-8 bytes (surrogate escapes) must not break normalization
+    weird = b"bad\xff.txt".decode("utf-8", "surrogateescape")
+    assert lookup_child(make_dir([weird]), weird)[0] == weird
+
+    with pytest.raises(KeyError):
+        lookup_child(only_nfc, "no-such-file.txt")
+
+    with pytest.raises(KeyError):
+        lookup_child(VFSNode(1), "anything")  # not a directory
+
+
+# The ACL xattrs (see vfs.ACL_XATTRS) are emulated by the VFS, so we can test them by
+# calling the xattr methods with a fake item. Unlike the tests in archiver/mount_cmds_test.py,
+# this also works in environments where the kernel does not offer ACL xattrs on FUSE mounts
+# (e.g. mounts made inside a user namespace). They are only emulated on Linux:
+
+
+@skipif_not_linux
+def test_listxattr_acls():
+    names = item_listxattr(make_item())
+    assert sorted(names) == [b"system.posix_acl_access", b"system.posix_acl_default", b"user.foo"]
+
+
+@skipif_not_linux
+def test_listxattr_no_acls():
+    assert item_listxattr(make_item(acls=False)) == [b"user.foo"]
+
+
+@skipif_not_linux
+@pytest.mark.parametrize("numeric_ids", [False, True])
+def test_getxattr_acls(numeric_ids):
+    item = make_item()
+    access = item_getxattr(item, b"system.posix_acl_access", numeric_ids=numeric_ids)
+    assert access == acl_text_to_xattr(ACCESS_ACL, numeric_ids=numeric_ids)
+    default = item_getxattr(item, b"system.posix_acl_default", numeric_ids=numeric_ids)
+    assert default == acl_text_to_xattr(DEFAULT_ACL, numeric_ids=numeric_ids)
+    assert item_getxattr(item, b"user.foo") == b"bar"  # "normal" xattrs still work
+
+
+@skipif_not_linux
+def test_getxattr_acl_missing():
+    with pytest.raises(KeyError):
+        item_getxattr(make_item(acls=False), b"system.posix_acl_access")
+
+
+@skipif_not_linux
+def test_getxattr_acl_broken():
+    with pytest.raises(ValueError):
+        item_getxattr(make_item(acl_access=BROKEN_ACL), b"system.posix_acl_access")
+
+
+def test_getxattr_missing():
+    with pytest.raises(KeyError):
+        item_getxattr(make_item(), b"user.nope")
diff --git a/src/borg/vfs.py b/src/borg/vfs.py
new file mode 100644
index 0000000000..e15a091b72
--- /dev/null
+++ b/src/borg/vfs.py
@@ -0,0 +1,723 @@
+"""
+The read-only "archive as a file system" implementation, shared by all its users.
+
+borg presents archive contents as a browsable directory tree in several places:
+
+- ``borg mount`` via mfusepy (high-level FUSE 2/3), see hlfuse.py
+- ``borg mount`` via llfuse / pyfuse3 (low-level FUSE 2/3), see fuse.py
+- ``borg webdav`` (WebDAV / HTTP server), see webdav.py
+
+They all need the same thing: build a directory tree from the items of the selected
+archives, map borg item metadata to file attributes and read file content out of the
+chunk lists. That is what this module does - the modules above are thin protocol
+adapters over it, so "what an archive looks like as a file system" is defined here
+and only here.
+
+Threading: an ArchiveVFS is not thread-safe by itself. Users that access it from
+multiple threads (webdav) pass a lock; all repository access (building an archive's
+tree, fetching chunks) happens while holding it, because borgstore connections are
+not thread-safe. The lock is also used to keep the repository lock refresher off the
+repository while a request is being served, see storelocking.LockRefresher.
+"""
+
+import hashlib
+import os
+import stat
+import threading
+import time
+import unicodedata
+from collections import Counter
+from typing import NamedTuple
+
+from .archive import Archive, DownloadPipeline, get_item_uid_gid
+from .constants import ROBJ_FILE_STREAM
+from .hashindex import FuseVersionsIndex
+from .helpers import Error, bin_to_hex, format_file_size, remove_surrogates
+from .helpers import msgpack
+from .helpers.lrucache import LRUCache
+from .item import ChunkListEntry, Item
+from .logger import create_logger
+from .platform import acl_text_to_xattr, uid2user, gid2group
+from .platformflags import is_darwin, is_linux, is_win32
+
+logger = create_logger()
+
+# on Linux, the kernel exposes POSIX ACLs via these special, binary encoded xattrs.
+# maps the xattr name to the borg item attribute holding the ACL text.
+# empty on platforms we can not do this for, so the mount just does not offer these xattrs there.
+ACL_XATTRS = {b"system.posix_acl_access": "acl_access", b"system.posix_acl_default": "acl_default"} if is_linux else {}
+
+DEFAULT_DIR_MODE = 0o40755
+
+# size of the cache of unpacked items (see ArchiveVFS.get_item).
+# note: such an item can be rather large - Item.chunks can be a long list.
+ITEM_CACHE_SIZE = 8
+
+
+class ChunkMissing(Exception):
+    """A chunk of a file's content is missing in the repository."""
+
+
+class VFSOptions:
+    """How the archive contents shall be presented; the defaults suit a plain, read-only view.
+
+    *versions* merges all selected archives into one tree, showing each file as a
+    directory that contains all its versions (see ArchiveVFS._load_archive).
+    *numeric_ids*, *uid_forced*, *gid_forced* and *umask* control the ownership and
+    permissions mapping, *strip_components* and *item_filter* which items are shown,
+    *allow_damaged_files* whether reads of files with missing chunks return zeros
+    instead of failing, and *dir_item* is the item used for synthesized directories.
+    """
+
+    def __init__(
+        self,
+        *,
+        versions=False,
+        numeric_ids=False,
+        uid_forced=None,
+        gid_forced=None,
+        umask=0,
+        allow_damaged_files=False,
+        strip_components=0,
+        item_filter=None,
+        dir_item=None,
+    ):
+        self.versions = versions
+        self.numeric_ids = numeric_ids
+        self.uid_forced = uid_forced
+        self.gid_forced = gid_forced
+        self.umask = umask
+        self.allow_damaged_files = allow_damaged_files
+        self.strip_components = strip_components
+        self.item_filter = item_filter
+        self.dir_item = dir_item
+
+
+class Attrs(NamedTuple):
+    """The file attributes of a node, as far as an archived item defines them."""
+
+    ino: int
+    mode: int
+    nlink: int
+    uid: int
+    gid: int
+    rdev: int
+    size: int
+    mtime_ns: int
+    atime_ns: int
+    ctime_ns: int
+    birthtime_ns: int
+
+
+class VFSNode:
+    """A node of the file system tree: a directory (children is a dict) or a leaf.
+
+    A node only knows its place in the tree - the item metadata lives in ArchiveVFS,
+    keyed by the inode number. Hard links to the same archived file are separate nodes
+    sharing one inode number (and thus one item).
+    """
+
+    __slots__ = ("ino", "parent", "children", "nfc_names")
+
+    def __init__(self, ino, parent=None, *, is_dir=False):
+        self.ino = ino
+        self.parent = parent
+        self.children = {} if is_dir else None  # name (str) -> VFSNode
+        self.nfc_names = None  # lazily built by ArchiveVFS.lookup(), see there
+
+    @property
+    def is_dir(self):
+        return self.children is not None
+
+
+def nfc(name):
+    """NFC-normalize *name*; names with surrogates (non-UTF-8 bytes) pass through unchanged."""
+    return unicodedata.normalize("NFC", name)
+
+
+def item_listxattr(item):
+    """Return the xattr names (bytes) of *item*."""
+    names = list(item.get("xattrs", {}).keys())
+    # expose the archived POSIX ACLs, so e.g. getfacl or tools copying from the mount can read them.
+    names.extend(xattr_name for xattr_name, attr in ACL_XATTRS.items() if attr in item)
+    return names
+
+
+def item_getxattr(item, name, *, numeric_ids=False):
+    """Return the value of xattr *name* (bytes) of *item*.
+
+    Raises KeyError if the item does not have that xattr, ValueError if an archived
+    ACL can not be converted to the binary xattr representation.
+    """
+    if name in ACL_XATTRS:
+        acl = item.get(ACL_XATTRS[name])
+        if acl is None:
+            raise KeyError(name)
+        return acl_text_to_xattr(acl, numeric_ids=numeric_ids)  # raises ValueError
+    return item.get("xattrs", {})[name] or b""
+
+
+def lookup_child(node, name):
+    """Return (child_name, child_node) for *name* in directory *node*. Raises KeyError.
+
+    An exact match always wins. If there is none, we retry comparing Unicode NFC forms:
+    file systems disagree about normalization (macOS decomposes, so its clients ask for
+    "gru<combining diaeresis>..." where the archive stores "grü..."), and an exact-only
+    lookup would answer "not found" for a file the client just saw in a listing.
+
+    The NFC index is built lazily (only for directories that actually get such a request)
+    and skips names that are ambiguous, i.e. where several different names share one NFC
+    form - those keep requiring an exact match, so we never silently serve the wrong file.
+    """
+    children = node.children
+    if children is None:
+        raise KeyError(name)  # not a directory
+    try:
+        return name, children[name]
+    except KeyError:
+        pass
+    if node.nfc_names is None:
+        index = {}
+        for child_name in children:
+            key = nfc(child_name)
+            index[key] = None if key in index else child_name  # None marks an ambiguous key
+        node.nfc_names = index
+    child_name = node.nfc_names.get(nfc(name))
+    if child_name is None:
+        raise KeyError(name)
+    return child_name, children[child_name]
+
+
+def versioned_name(name, version):
+    """Build the name a file has in the versions view, e.g. "file.00001.txt"."""
+    # keep the original extension at the end, to avoid confusing tools
+    name, ext = os.path.splitext(name)
+    return f"{name}.{version:05d}{ext}"
+
+
+class ArchiveVFS:
+    """A read-only file system view of the archives selected by *args*.
+
+    The root directory has one directory per (deduplicated) archive name; each of
+    these archive trees is built when it is first accessed - building it means
+    reading the whole item metadata stream of that archive, which takes a while for
+    a big archive. In the versions view, the contents of all selected archives are
+    merged into one tree instead, which is built completely upfront.
+
+    Directories are addressable by inode number (get_node); everything else is
+    reached by looking up names in a directory node (lookup / resolve).
+    """
+
+    def __init__(self, manifest, args, repository=None, *, lock=None, options=None):
+        self.manifest = manifest
+        self.repository = repository if repository is not None else manifest.repository
+        self.args = args
+        self.options = options if options is not None else VFSOptions()
+        self.lock = lock if lock is not None else threading.RLock()
+        self.pipeline = DownloadPipeline(self.repository, manifest.repo_objs)
+        self.reader = DataReader(self.pipeline, lock=self.lock, allow_damaged_files=self.options.allow_damaged_files)
+        # the ids of the mounting user are what items without a usable uid/gid fall back to.
+        # note: borg webdav also runs on Windows, which has no uid/gid (nor os.getuid).
+        self.default_uid = 0 if is_win32 else os.getuid()
+        self.default_gid = 0 if is_win32 else os.getgid()
+        self.archives = {}  # display name -> ArchiveInfo
+        self.root_mtime = int(time.time() * 1e9)  # replaced by create_filesystem()
+        self._ino_count = 0
+        self._items = {}  # inode number -> msgpacked item (None -> options.dir_item)
+        self._item_cache = LRUCache(capacity=ITEM_CACHE_SIZE)  # inode number -> Item
+        self._nodes = {}  # inode number -> VFSNode, for directories only
+        self._nlinks = {}  # inode number -> link count, for hard linked items only
+        self._pending = {}  # VFSNode -> ArchiveInfo, archives whose tree is not built yet
+        self._versions_index = None
+        self.root = self._new_node(None, is_dir=True)
+        assert self.root.ino == 1  # the root inode number is 1 by convention
+
+    # -- setting up ---------------------------------------------------------------
+
+    def create_filesystem(self):
+        """Populate the root directory (this also selects the archives to be shown)."""
+        archives = self.manifest.archives.list_considering(self.args)
+        # archives of a series all have the same name, so make the display names unique
+        name_counter = Counter(archive.name for archive in archives)
+        for archive in archives:
+            name = archive.name
+            if name_counter[name] > 1:
+                name += f"-{bin_to_hex(archive.id):.8}"
+            self.archives[name] = archive
+        timestamps = [archive.ts for archive in archives]
+        self.root_mtime = int(max(timestamps).timestamp() * 1e9) if timestamps else int(time.time() * 1e9)
+        self._set_item(self.root.ino, self._dir_item(self.root_mtime))
+        if self.options.versions:
+            # the versions view merges all archives into one tree, so there is nothing
+            # to defer: build it now.
+            self._versions_index = FuseVersionsIndex()
+            for archive in self.archives.values():
+                self._load_archive(archive, self.root)
+        else:
+            for name, archive in self.archives.items():
+                node = self._new_node(self.root, is_dir=True)
+                self._set_item(node.ino, self._dir_item(int(archive.ts.timestamp() * 1e9)))
+                self.root.children[name] = node
+                self._pending[node] = archive
+
+    def _dir_item(self, mtime):
+        """Return the item used for a synthesized directory, with mtime *mtime*."""
+        item = Item(internal_dict=self._default_dir().as_dict())
+        item.mtime = mtime
+        return item
+
+    def _default_dir(self):
+        """The item used for all synthesized directories that keep the default mtime."""
+        if self.options.dir_item is None:
+            uid = self.options.uid_forced if self.options.uid_forced is not None else self.default_uid
+            gid = self.options.gid_forced if self.options.gid_forced is not None else self.default_gid
+            self.options.dir_item = Item(
+                mode=DEFAULT_DIR_MODE & ~self.options.umask, mtime=int(time.time() * 1e9), uid=uid, gid=gid
+            )
+        return self.options.dir_item
+
+    # -- building the tree --------------------------------------------------------
+
+    def ensure_loaded(self, node):
+        """Build the tree of an archive directory, if that did not happen yet."""
+        if node not in self._pending:
+            return  # not an archive directory, or its tree is complete
+        with self.lock:
+            archive = self._pending.get(node)
+            if archive is None:
+                return  # another thread built it while we waited for the lock
+            self._load_archive(archive, node)
+            # only now (with the tree complete) let other threads past the check above.
+            del self._pending[node]
+
+    def _load_archive(self, archive_info, root):
+        """Add the items of one archive to the tree below *root*."""
+        t0 = time.perf_counter()
+        archive = Archive(self.manifest, archive_info.id)
+        archive_mtime = int(archive_info.ts.timestamp() * 1e9)
+        strip_components = self.options.strip_components
+        versions = self.options.versions
+        hardlinks = {}  # hlid -> node of the first item with that hlid
+        for item in archive.iter_items(self.options.item_filter):
+            if strip_components:
+                item.path = os.sep.join(item.path.split(os.sep)[strip_components:])
+            segments = [segment for segment in item.path.split("/") if segment]
+            if not segments:
+                continue
+            parent = self._make_dirs(root, segments[:-1], mtime=archive_mtime)
+            name = segments[-1]
+            if stat.S_ISDIR(item.mode):
+                # the directory may have been synthesized already (as the parent of an
+                # item that came first): then just replace its item metadata.
+                node = parent.children.get(name)
+                if node is None or not node.is_dir:
+                    node = self._new_node(parent, is_dir=True)
+                    parent.children[name] = node
+                self._set_item(node.ino, item)
+                continue
+            if versions:
+                # the file name becomes a directory that contains all versions of that file
+                parent = self._make_dirs(parent, [name])
+                version = self._file_version(item, item.path)
+                if version is not None:  # a regular file, with contents
+                    name = versioned_name(name, version)
+            hlid = item.get("hlid")
+            if hlid is not None:
+                first = hardlinks.get(hlid)
+                if first is not None:
+                    # another link to a file we already have: share its inode (and item),
+                    # so the mount shows the hard link as such.
+                    self._nlinks[first.ino] = self._nlinks.get(first.ino, 1) + 1
+                    parent.children[name] = VFSNode(first.ino, parent)
+                    continue
+            node = self._new_node(parent, is_dir=False)
+            self._set_item(node.ino, item)
+            parent.children[name] = node
+            if hlid is not None:
+                hardlinks[hlid] = node
+        duration = time.perf_counter() - t0
+        logger.debug("vfs: built the tree of archive %s in %.1f s", remove_surrogates(archive_info.name), duration)
+
+    def _make_dirs(self, node, segments, mtime=None):
+        """Return the directory at *segments* below *node*, synthesizing what is missing.
+
+        A synthesized directory gets an item of its own if *mtime* is given (so it can show
+        the time of the archive it belongs to) - otherwise it shares the default directory
+        item, which is what the many intermediate directories of the versions view do.
+        """
+        for segment in segments:
+            child = node.children.get(segment)
+            if child is None or not child.is_dir:
+                # a directory we have not seen an item for (yet): synthesize it.
+                child = self._new_node(node, is_dir=True)
+                if mtime is not None:
+                    self._set_item(child.ino, self._dir_item(mtime))
+                node.children[segment] = child
+            node = child
+        return node
+
+    def _new_node(self, parent, *, is_dir):
+        self._ino_count += 1
+        node = VFSNode(self._ino_count, parent, is_dir=is_dir)
+        if is_dir:
+            self._nodes[node.ino] = node
+        return node
+
+    def _file_version(self, item, path):
+        """Return the version number of *item* in the versions view (None if it has no contents).
+
+        Files are versioned by their content: a file keeps its version number as long as
+        its chunk list does not change from one archive to the next.
+        """
+        if "chunks" not in item:
+            return None
+        # note: using sha256 here because nowadays it is often hw accelerated.
+        #       shortening the hashes to 16 bytes to save some memory.
+        file_id = hashlib.sha256(path.encode("utf-8", "surrogateescape")).digest()[:16]
+        current_version, previous_id = self._versions_index.get(file_id, (0, None))
+        contents_id = hashlib.sha256(b"".join(chunk_id for chunk_id, _ in item.chunks)).digest()[:16]
+        if contents_id != previous_id:
+            current_version += 1
+            self._versions_index[file_id] = current_version, contents_id
+        return current_version
+
+    # -- looking things up --------------------------------------------------------
+
+    def get_node(self, ino):
+        """Return the directory node with inode number *ino*. Raises KeyError."""
+        return self._nodes[ino]
+
+    def lookup(self, node, name):
+        """Return (canonical_name, child_node) for *name* in directory *node*. Raises KeyError."""
+        self.ensure_loaded(node)
+        return lookup_child(node, name)
+
+    def resolve(self, segments, node=None):
+        """Resolve path *segments* to (canonical_segments, node). Raises KeyError.
+
+        *canonical_segments* are the names as they are stored in the archive, which may
+        differ from what was asked for, see lookup().
+        """
+        node = self.root if node is None else node
+        canonical = []
+        for segment in segments:
+            name, node = self.lookup(node, segment)
+            canonical.append(name)
+        return canonical, node
+
+    def children(self, node):
+        """Return the [(name, child_node), ...] of directory *node*."""
+        self.ensure_loaded(node)
+        return list(node.children.items())
+
+    def get_item(self, ino):
+        """Return the item of the node with inode number *ino*."""
+        packed = self._items.get(ino)
+        if packed is None:
+            return self._default_dir()  # a synthesized directory
+        # note: two threads can miss on the same inode and both unpack and store the item.
+        # that is a bit of duplicated work, but both get a valid item.
+        item = self._item_cache.get(ino)
+        if item is None:
+            item = Item(internal_dict=msgpack.unpackb(packed))
+            if "chunks" in item:  # msgpack does not know about our namedtuple
+                item.chunks = [ChunkListEntry(*chunk) for chunk in item.chunks]
+            self._item_cache[ino] = item
+        return item
+
+    def _set_item(self, ino, item):
+        item_dict = item.as_dict()
+        # the path is already encoded in the tree structure, so drop it to save memory.
+        item_dict.pop("path", None)
+        self._items[ino] = msgpack.packb(item_dict)
+        self._item_cache.pop(ino, None)
+
+    # -- item metadata ------------------------------------------------------------
+
+    def attrs(self, ino):
+        """Return the file attributes of the node with inode number *ino*."""
+        item = self.get_item(ino)
+        options = self.options
+        mode = item.mode & ~options.umask
+        nlink = self._nlinks.get(ino, 1)
+        if stat.S_ISDIR(mode):
+            nlink = max(nlink, 2)  # every directory has at least "." and its entry in the parent
+        uid, gid = get_item_uid_gid(
+            item,
+            numeric=options.numeric_ids,
+            uid_default=self.default_uid,
+            gid_default=self.default_gid,
+            uid_forced=options.uid_forced,
+            gid_forced=options.gid_forced,
+        )
+        # note: old archives only have mtime (not atime nor ctime nor birthtime)
+        mtime_ns = item.mtime
+        return Attrs(
+            ino=ino,
+            mode=mode,
+            nlink=nlink,
+            uid=uid,
+            gid=gid,
+            rdev=item.get("rdev", 0),
+            size=item.get_size(),
+            mtime_ns=mtime_ns,
+            atime_ns=item.get("atime", mtime_ns),
+            ctime_ns=item.get("ctime", mtime_ns),
+            birthtime_ns=item.get("birthtime", mtime_ns),
+        )
+
+    def listxattr(self, ino):
+        """Return the xattr names (bytes) of the node with inode number *ino*."""
+        return item_listxattr(self.get_item(ino))
+
+    def getxattr(self, ino, name):
+        """Return the value of xattr *name* (bytes) of the node with inode number *ino*.
+
+        Raises KeyError if the item does not have that xattr, ValueError if an archived
+        ACL can not be converted to the binary xattr representation.
+        """
+        return item_getxattr(self.get_item(ino), name, numeric_ids=self.options.numeric_ids)
+
+    def readlink(self, ino):
+        """Return the target of the symlink with inode number *ino*."""
+        return self.get_item(ino).target
+
+    def log_stats(self):
+        """Log some statistics about this file system (used by the SIGUSR1/SIGINFO handler)."""
+        logger.debug(
+            "vfs: %d inodes, %d items (%s), %d archives not loaded yet",
+            self._ino_count,
+            len(self._items),
+            format_file_size(sum(len(packed) for packed in self._items.values())),
+            len(self._pending),
+        )
+        data_cache = self.reader.data_cache
+        logger.debug(
+            "vfs: data cache: %d/%d entries, %s",
+            len(data_cache),
+            data_cache._capacity,
+            format_file_size(sum(len(chunk) for _, chunk in data_cache.items())),
+        )
+
+    # -- file contents ------------------------------------------------------------
+
+    def read(self, ino, offset, size, *, pos_key=None):
+        """Return *size* bytes at *offset* of the file with inode number *ino*."""
+        item = self.get_item(ino)
+        return self.reader.read(item.get("chunks") or [], offset, size, pos_key=pos_key)
+
+
+class DataReader:
+    """Reads file content out of the chunk lists of items.
+
+    Recently used chunks are kept decrypted in an in-memory cache, so the many small,
+    sequential reads a file system does for one big file do not fetch and decrypt the
+    same chunk over and over. This complements, but does not duplicate, the repository's
+    own pack cache: that one caches the raw (still encrypted and compressed) on-disk
+    bytes, whereas we cache the output of the per-chunk RepoObj.parse() that runs on
+    every fetch even on a pack cache hit. So the two layers save different work
+    (repository I/O vs. per-chunk decrypt + decompress).
+
+    All repository access happens under *lock* (borgstore connections are not thread-safe),
+    but the data is returned to the caller outside of it, so a slow consumer (e.g. a slow
+    network client) does not block other users of the same repository. The caches are
+    thread-safe by themselves, see LRUCache.
+    """
+
+    # how many chunk positions to remember for sequential reads (1 per open file)
+    POSITIONS = 8
+
+    def __init__(self, pipeline, *, lock=None, capacity=None, allow_damaged_files=False):
+        self.pipeline = pipeline
+        self.lock = lock if lock is not None else threading.RLock()
+        capacity = data_cache_capacity() if capacity is None else capacity
+        logger.debug("vfs: data cache capacity: %d chunks", capacity)
+        self.data_cache = LRUCache(capacity=capacity)  # chunk id -> decrypted chunk data
+        self.allow_damaged_files = allow_damaged_files
+        # remembers where a sequential reader is within the chunk list, so a read does
+        # not have to walk the list from the beginning again (that would be quadratic).
+        self._last_pos = LRUCache(capacity=self.POSITIONS)
+
+    def read(self, chunks, offset, size, *, pos_key=None):
+        """Return *size* bytes at *offset* of the content described by the *chunks* list."""
+        return b"".join(self.iter_data(chunks, offset, size, pos_key=pos_key))
+
+    def iter_data(self, chunks, offset, size, *, pos_key=None):
+        """Yield the bytes at [offset, offset+size) of the content described by *chunks*.
+
+        *pos_key* identifies the reader (e.g. a file handle) for the sequential-read
+        optimization. Raises ChunkMissing if a chunk is not in the repository (unless
+        allow_damaged_files is set - then all-zero data is returned for it).
+        """
+        if size <= 0:
+            return
+        chunk_no, chunk_offset = self._get_pos(pos_key) if pos_key is not None else (0, 0)
+        if chunk_offset > offset:
+            # this is not a sequential read, so we lost track and start from the beginning again
+            chunk_no, chunk_offset = 0, 0
+        offset -= chunk_offset
+        # note: using index iteration to avoid frequently copying big (sub)lists by slicing
+        for idx in range(chunk_no, len(chunks)):
+            entry = chunks[idx]
+            if entry.size <= offset:
+                offset -= entry.size
+                chunk_offset += entry.size
+                chunk_no += 1
+                continue
+            n = min(size, entry.size - offset)
+            data = self._get_chunk(entry, fully_read=offset + n >= entry.size)
+            yield data[offset : offset + n]
+            offset = 0
+            size -= n
+            if not size:
+                if pos_key is not None:
+                    self._set_pos(pos_key, (chunk_no, chunk_offset))
+                break
+
+    def _get_chunk(self, entry, fully_read=False):
+        """Return the decrypted data of chunk *entry*, using (and updating) the data cache.
+
+        A chunk that is *fully_read* is not worth keeping: a sequential reader is done
+        with it, so it is dropped from the cache instead of evicting something else.
+        """
+        with self.lock:
+            try:
+                data = self.data_cache[entry.id]
+            except KeyError:
+                pass
+            else:
+                if fully_read:
+                    del self.data_cache[entry.id]
+                return data
+            data = next(
+                self.pipeline.fetch_many([entry], ro_type=ROBJ_FILE_STREAM, replacement_chunk=self.allow_damaged_files)
+            )
+            if data is None:
+                raise ChunkMissing(entry.id)
+            if not fully_read:
+                self.data_cache[entry.id] = data
+            return data
+
+    # the read positions are only a hint: a read that starts before the remembered
+    # position just starts over, so a racing update costs at most a rescan of the
+    # chunk list.
+
+    def _get_pos(self, pos_key):
+        return self._last_pos.get(pos_key, (0, 0))
+
+    def _set_pos(self, pos_key, pos):
+        self._last_pos[pos_key] = pos
+
+    def forget(self, pos_key):
+        """Forget the read position of *pos_key* (call this when a file is closed)."""
+        self._last_pos.pop(pos_key, None)
+
+
+def data_cache_capacity():
+    # number of decrypted chunks to keep cached, see borg mount --help.
+    # default: number of CPUs (a small, non-zero value).
+    capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1))
+    return max(1, capacity)
+
+
+def build_item_filter(args):
+    """Build the item filter selecting the paths/patterns given on the command line."""
+    # lazy import: pulling in the archiver package at module import time would be heavy
+    # (it defines all subcommands) and risks an import cycle.
+    from .archiver._common import build_matcher, build_filter
+
+    # omitting args.pattern_roots here, restricting to paths only by cli args.paths:
+    matcher = build_matcher(getattr(args, "patterns", None) or [], getattr(args, "paths", None) or [])
+    return build_filter(matcher, getattr(args, "strip_components", 0))
+
+
+def pop_option(options, key, present, not_present, wanted_type, int_base=0):
+    """Remove mount option *key* from the *options* list and return its value."""
+    assert isinstance(options, list)  # we mutate this
+    for idx, option in enumerate(options):
+        if option == key:
+            options.pop(idx)
+            return present
+        if option.startswith(key + "="):
+            options.pop(idx)
+            value = option.split("=", 1)[1]
+            if wanted_type is bool:
+                v = value.lower()
+                if v in ("y", "yes", "true", "1"):
+                    return True
+                if v in ("n", "no", "false", "0"):
+                    return False
+                raise ValueError("unsupported value in option: %s" % option)
+            if wanted_type is int:
+                try:
+                    return int(value, base=int_base)
+                except ValueError:
+                    raise ValueError("unsupported value in option: %s" % option) from None
+            try:
+                return wanted_type(value)
+            except ValueError:
+                raise ValueError("unsupported value in option: %s" % option) from None
+    else:
+        return not_present
+
+
+def parse_mount_options(args, mountpoint, mount_options):
+    """Process the "borg mount" options; returns (libfuse options, VFSOptions).
+
+    The borg specific options are removed from the option list here and end up in the
+    VFSOptions, everything else is passed on to libfuse.
+    """
+    # default_permissions enables permission checking by the kernel. Without
+    # this, any umask (or uid/gid) would not have an effect and this could
+    # cause security issues if used with allow_other mount option.
+    # When not using allow_other or allow_root, access is limited to the
+    # mounting user anyway.
+    options = ["fsname=borgfs", "ro", "default_permissions"]
+    if mount_options:
+        options.extend(mount_options.split(","))
+    if is_darwin:
+        # macFUSE supports a volname mount option to give what finder displays on desktop / in directory list.
+        volname = pop_option(options, "volname", "", "", str)
+        # if the user did not specify it, we make something up,
+        # because otherwise it would be "macFUSE Volume 0 (Python)", #7690.
+        volname = volname or f"{os.path.basename(mountpoint)} (borgfs)"
+        options.append(f"volname={volname}")
+    ignore_permissions = pop_option(options, "ignore_permissions", True, False, bool)
+    if ignore_permissions:
+        # in case users have a use-case that requires NOT giving "default_permissions",
+        # this is enabled by the custom "ignore_permissions" mount option which just
+        # removes "default_permissions" again:
+        pop_option(options, "default_permissions", True, False, bool)
+    vfs_options = VFSOptions(
+        allow_damaged_files=pop_option(options, "allow_damaged_files", True, False, bool),
+        versions=pop_option(options, "versions", True, False, bool),
+        uid_forced=pop_option(options, "uid", None, None, int),
+        gid_forced=pop_option(options, "gid", None, None, int),
+        umask=pop_option(options, "umask", 0, 0, int, int_base=8),  # umask is octal, e.g. 222 or 0222
+        numeric_ids=getattr(args, "numeric_ids", False),
+        strip_components=getattr(args, "strip_components", 0),
+        item_filter=build_item_filter(args),
+    )
+    dir_uid = vfs_options.uid_forced if vfs_options.uid_forced is not None else os.getuid()
+    dir_gid = vfs_options.gid_forced if vfs_options.gid_forced is not None else os.getgid()
+    dir_user = uid2user(dir_uid)
+    dir_group = gid2group(dir_gid)
+    if not isinstance(dir_user, str):
+        raise Error(
+            f"uid {dir_uid} can not be resolved to a username. "
+            f"Please check that the corresponding user exists or do not specify a uid mount option."
+        )
+    if not isinstance(dir_group, str):
+        raise Error(
+            f"gid {dir_gid} can not be resolved to a group name. "
+            f"Please check that the corresponding group exists or do not specify a gid mount option."
+        )
+    vfs_options.dir_item = Item(
+        mode=DEFAULT_DIR_MODE & ~vfs_options.umask,
+        mtime=int(time.time() * 1e9),
+        user=dir_user,
+        group=dir_group,
+        uid=dir_uid,
+        gid=dir_gid,
+    )
+    return options, vfs_options
diff --git a/src/borg/webdav.py b/src/borg/webdav.py
index da2791d999..85435e2d3c 100644
--- a/src/borg/webdav.py
+++ b/src/borg/webdav.py
@@ -13,15 +13,14 @@
 
 import html
 import mimetypes
-import os
 import re
 import stat
 import tarfile
 import threading
-import unicodedata
 from datetime import datetime, timezone
 from email.utils import formatdate
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import NamedTuple
 from urllib.parse import parse_qs, quote, unquote_to_bytes
 from xml.etree import ElementTree as ET  # nosec B405 # only used to *build* response XML, never to parse input
 from xml.parsers import expat
@@ -29,177 +28,26 @@
 from . import __version__
 from .archive import Archive
 from .constants import *  # NOQA
-from .helpers import bin_to_hex, remove_surrogates, HardLinkManager
-from .helpers.lrucache import LRUCache
+from .helpers import remove_surrogates, HardLinkManager
 from .logger import create_logger
+from .vfs import ArchiveVFS, ChunkMissing, DEFAULT_DIR_MODE
 
 logger = create_logger(__name__)
 
-DEFAULT_DIR_MODE = 0o40755
 
+class Resource(NamedTuple):
+    """What this server needs to know about one file system object it serves.
 
-class Node:
-    """A node in an archive's directory tree: a directory (children is a dict) or a leaf."""
-
-    __slots__ = ("mode", "mtime", "size", "chunks", "target", "children", "nfc_names")
-
-    def __init__(self, mode, mtime=0, size=0, chunks=None, target=None, children=None):
-        self.mode = mode
-        self.mtime = mtime  # ns
-        self.size = size
-        self.chunks = chunks
-        self.target = target  # symlink target
-        self.children = children  # {name(str): Node} for directories, None otherwise
-        self.nfc_names = None  # lazily built by lookup_child(), see there
-
-    @property
-    def is_dir(self):
-        return self.children is not None
-
-
-def nfc(name):
-    """NFC-normalize *name*; names with surrogates (non-UTF-8 bytes) pass through unchanged."""
-    return unicodedata.normalize("NFC", name)
-
-
-def lookup_child(node, name):
-    """Return (child_name, child_node) for *name* in directory *node*. Raises KeyError.
-
-    An exact match always wins. If there is none, we retry comparing Unicode NFC forms:
-    file systems disagree about normalization (macOS decomposes, so its WebDAV client asks
-    for "gru<combining diaeresis>..." where the archive stores "grü..."), and an exact-only
-    lookup would answer 404 for a file the client just saw in our listing.
-
-    The NFC index is built lazily (only for directories that actually get such a request)
-    and skips names that are ambiguous, i.e. where several different names share one NFC
-    form - those keep requiring an exact match, so we never silently serve the wrong file.
-    """
-    try:
-        return name, node.children[name]
-    except KeyError:
-        pass
-    if node.nfc_names is None:
-        index = {}
-        for child_name in node.children:
-            key = nfc(child_name)
-            index[key] = None if key in index else child_name  # None marks an ambiguous key
-        node.nfc_names = index
-    child_name = node.nfc_names.get(nfc(name))
-    if child_name is None:
-        raise KeyError(name)
-    return child_name, node.children[child_name]
-
-
-class ArchiveVFS:
-    """Read-only view of the archives selected by the archive filter args.
-
-    The top level maps (deduplicated) archive names to lazily built directory trees.
-    All repository access happens under repo_lock, because borgstore connections are
-    not thread-safe.
+    The tree itself lives in the (shared) archive VFS - this is just the flat view
+    of one node of it that building a listing, a PROPFIND response or a download
+    needs, so we do not look up the same item metadata over and over.
     """
 
-    def __init__(self, manifest, args, repo_lock):
-        self.manifest = manifest
-        self.repo_lock = repo_lock
-        archives = manifest.archives.list_considering(args)
-        # deduplicate archive names (archives of a series all share the same name)
-        name_counter = {}
-        for archive in archives:
-            name_counter[archive.name] = name_counter.get(archive.name, 0) + 1
-        self.archives = {}  # display name -> ArchiveInfo
-        for archive in archives:
-            name = archive.name
-            if name_counter[name] > 1:
-                name += f"-{bin_to_hex(archive.id):.8}"
-            self.archives[name] = archive
-        self._trees = {}  # display name -> (root Node, DownloadPipeline)
-        self.nfc_archives = None  # lazily built by lookup_archive(), see there
-        timestamps = [archive.ts for archive in archives]
-        self.root_mtime = int(max(timestamps).timestamp() * 1e9) if timestamps else 0
-
-    def get_root(self, name):
-        """Return (root Node, pipeline) for archive *name*, building the tree on first access."""
-        try:
-            return self._trees[name]
-        except KeyError:
-            pass
-        archive_info = self.archives[name]  # may raise KeyError -> 404
-        with self.repo_lock:
-            if name not in self._trees:  # re-check under lock
-                self._trees[name] = self._build_tree(archive_info)
-        return self._trees[name]
-
-    def _build_tree(self, archive_info):
-        logger.debug("webdav: building tree for archive %s ...", remove_surrogates(archive_info.name))
-        archive = Archive(self.manifest, archive_info.id)
-        archive_mtime = int(archive_info.ts.timestamp() * 1e9)
-        root = Node(DEFAULT_DIR_MODE, mtime=archive_mtime, children={})
-        for item in archive.iter_items():
-            segments = [s for s in item.path.split("/") if s]
-            if not segments:
-                continue
-            node = root
-            for segment in segments[:-1]:
-                child = node.children.get(segment)
-                if child is None or not child.is_dir:
-                    # intermediate directory not (yet) seen as an item: synthesize it
-                    child = Node(DEFAULT_DIR_MODE, mtime=archive_mtime, children={})
-                    node.children[segment] = child
-                node = child
-            name = segments[-1]
-            if stat.S_ISDIR(item.mode):
-                existing = node.children.get(name)
-                if existing is not None and existing.is_dir:
-                    # was synthesized before the dir item was seen: update metadata in place
-                    existing.mode = item.mode
-                    existing.mtime = item.mtime
-                else:
-                    node.children[name] = Node(item.mode, mtime=item.mtime, children={})
-            else:
-                node.children[name] = Node(
-                    item.mode,
-                    mtime=item.mtime,
-                    size=item.get_size(),
-                    chunks=item.get("chunks"),
-                    target=item.get("target"),
-                )
-        return root, archive.pipeline
-
-    def resolve(self, segments):
-        """Resolve path segments (first one is the archive name) to (Node, pipeline, canonical).
-
-        *canonical* are the segments as they are actually stored in the archive: a client may
-        address a path in a different Unicode normalization than the archive uses (see
-        lookup_child()), and the stored spelling is what path-based operations (e.g. the tar
-        download, which matches item paths) have to use.
-
-        Raises KeyError if not found.
-        """
-        archive_name = self.lookup_archive(segments[0])
-        root, pipeline = self.get_root(archive_name)
-        node = root
-        canonical = [archive_name]
-        for segment in segments[1:]:
-            if not node.is_dir:
-                raise KeyError(segment)
-            name, node = lookup_child(node, segment)
-            canonical.append(name)
-        return node, pipeline, canonical
-
-    def lookup_archive(self, name):
-        """Return the stored archive name matching *name*. Raises KeyError if there is none."""
-        if name in self.archives:
-            return name
-        if self.nfc_archives is None:
-            index = {}
-            for archive_name in self.archives:
-                key = nfc(archive_name)
-                index[key] = None if key in index else archive_name
-            self.nfc_archives = index
-        archive_name = self.nfc_archives.get(nfc(name))
-        if archive_name is None:
-            raise KeyError(name)
-        return archive_name
+    is_dir: bool
+    mode: int
+    mtime: int  # ns
+    size: int
+    target: str | None  # symlink target
 
 
 def strip_crlf(value):
@@ -240,9 +88,9 @@ def guess_content_type(name):
     return mimetypes.guess_type(remove_surrogates(name), strict=False)[0] or "application/octet-stream"
 
 
-def make_etag(node):
+def make_etag(res):
     # archive contents are immutable, so mtime+size identify the content well enough.
-    return f'"{node.mtime:x}-{node.size:x}"'
+    return f'"{res.mtime:x}-{res.size:x}"'
 
 
 def parse_byte_range(header, size):
@@ -357,33 +205,33 @@ def start_element(name, attrs):
     return "allprop", None  # allprop, maybe with an include element
 
 
-def make_prop_element(tag, name, node):
-    """Build the XML element for live property *tag* of resource *node*.
+def make_prop_element(tag, name, res):
+    """Build the XML element for live property *tag* of resource *res*.
 
     Returns None if the property is not defined for this resource.
     """
     elem = ET.Element(tag)
     if tag == "{DAV:}resourcetype":
-        if node.is_dir:
+        if res.is_dir:
             ET.SubElement(elem, "{DAV:}collection")
     elif tag == "{DAV:}displayname":
         elem.text = remove_surrogates(name)
     elif tag == "{DAV:}getlastmodified":
-        elem.text = http_date(node.mtime)
+        elem.text = http_date(res.mtime)
     elif tag == "{DAV:}creationdate":
-        elem.text = iso8601(node.mtime)
+        elem.text = iso8601(res.mtime)
     elif tag == "{DAV:}getcontentlength":
-        if node.is_dir:
+        if res.is_dir:
             return None
-        elem.text = str(node.size)
+        elem.text = str(res.size)
     elif tag == "{DAV:}getcontenttype":
-        if node.is_dir:
+        if res.is_dir:
             return None
         elem.text = guess_content_type(name)
     elif tag == "{DAV:}getetag":
-        if node.is_dir:
+        if res.is_dir:
             return None
-        elem.text = make_etag(node)
+        elem.text = make_etag(res)
     elif tag in ("{DAV:}supportedlock", "{DAV:}lockdiscovery"):
         pass  # empty elements: locking is not supported
     else:
@@ -394,22 +242,22 @@ def make_prop_element(tag, name, node):
 def render_multistatus(resources, mode, requested):
     """Render a PROPFIND result as a multistatus XML document (bytes).
 
-    *resources* is a list of (href, displayname, node) tuples, *mode* / *requested*
+    *resources* is a list of (href, displayname, Resource) tuples, *mode* / *requested*
     are the parse_propfind() results.
     """
     multistatus = ET.Element("{DAV:}multistatus")
-    for href, name, node in resources:
+    for href, name, res in resources:
         response = ET.SubElement(multistatus, "{DAV:}response")
         ET.SubElement(response, "{DAV:}href").text = href
         found = ET.Element("{DAV:}prop")
         missing = ET.Element("{DAV:}prop")
         if mode == "propname":
             for tag in DAV_PROPS:
-                if make_prop_element(tag, name, node) is not None:
+                if make_prop_element(tag, name, res) is not None:
                     ET.SubElement(found, tag)
         else:
             for tag in DAV_PROPS if mode == "allprop" else requested:
-                elem = make_prop_element(tag, name, node)
+                elem = make_prop_element(tag, name, res)
                 if elem is not None:
                     found.append(elem)
                 else:
@@ -549,9 +397,7 @@ class WebDAVHandler(BaseHTTPRequestHandler):
     sys_version = ""  # do not tell clients about the python version we use
 
     # set on the handler class by make_server():
-    vfs = None
-    repo_lock = None
-    data_cache = None  # LRUCache: chunk id -> decrypted chunk data, shared across requests
+    vfs = None  # the shared archive VFS, see vfs.py
 
     def version_string(self):
         # the base class would append sys_version, giving a trailing space if it is empty.
@@ -652,11 +498,12 @@ def _handle_get_head(self, head):
         try:
             # use the canonical (as stored in the archive) segments from here on, so that a
             # request in a different Unicode normalization still names the stored items.
-            node, pipeline, segments = self.vfs.resolve(segments)
+            segments, node = self.vfs.resolve(segments)
         except KeyError:
             self.send_error(404)
             return
-        if node.is_dir:
+        res = self._resource(node)
+        if res.is_dir:
             if not dir_syntax:
                 self._redirect_to_dir(segments)
                 return
@@ -664,15 +511,22 @@ def _handle_get_head(self, head):
                 self._send_tar(segments, head)
                 return
             self._send_dir_listing(segments, node, head)
-        elif stat.S_ISREG(node.mode):
-            self._send_file(segments[-1], node, pipeline, head)
-        elif stat.S_ISLNK(node.mode):
+        elif stat.S_ISREG(res.mode):
+            self._send_file(segments[-1], node, res, head)
+        elif stat.S_ISLNK(res.mode):
             self.send_error(
-                403, explain=f"symbolic link (target: {remove_surrogates(node.target or '?')}), not downloadable"
+                403, explain=f"symbolic link (target: {remove_surrogates(res.target or '?')}), not downloadable"
             )
         else:
             self.send_error(403, explain="special file, not downloadable")
 
+    def _resource(self, node):
+        """Return the Resource describing *node*."""
+        item = self.vfs.get_item(node.ino)
+        return Resource(
+            is_dir=node.is_dir, mode=item.mode, mtime=item.mtime, size=item.get_size(), target=item.get("target")
+        )
+
     def _handle_propfind(self):
         body = self._read_body()
         if body is None:
@@ -704,29 +558,32 @@ def _handle_propfind(self):
         self.wfile.write(result)
 
     def _propfind_resources(self, segments, depth):
-        """Return the [(href, displayname, node), ...] a PROPFIND on *segments* refers to."""
+        """Return the [(href, displayname, Resource), ...] a PROPFIND on *segments* refers to."""
         resources = []
         if not segments:  # server root: the list of archives
-            resources.append(("/", "/", Node(DEFAULT_DIR_MODE, mtime=self.vfs.root_mtime, children={})))
+            resources.append(("/", "/", Resource(True, DEFAULT_DIR_MODE, self.vfs.root_mtime, 0, None)))
             if depth == "1":
                 for name in sorted(self.vfs.archives):
                     archive_info = self.vfs.archives[name]
-                    node = Node(DEFAULT_DIR_MODE, mtime=int(archive_info.ts.timestamp() * 1e9), children={})
-                    resources.append(("/" + encode_path(name) + "/", name, node))
+                    mtime_ns = int(archive_info.ts.timestamp() * 1e9)
+                    res = Resource(True, DEFAULT_DIR_MODE, mtime_ns, 0, None)
+                    resources.append(("/" + encode_path(name) + "/", name, res))
             return resources
-        node, _, segments = self.vfs.resolve(segments)  # may raise KeyError
-        if not (node.is_dir or stat.S_ISREG(node.mode)):
+        segments, node = self.vfs.resolve(segments)  # may raise KeyError
+        res = self._resource(node)
+        if not (res.is_dir or stat.S_ISREG(res.mode)):
             raise KeyError(segments[-1])  # symlinks and special files are not exposed via WebDAV
         base = "/" + "/".join(encode_path(s) for s in segments)
-        if not node.is_dir:
-            return [(base, segments[-1], node)]
-        resources.append((base + "/", segments[-1], node))
+        if not res.is_dir:
+            return [(base, segments[-1], res)]
+        resources.append((base + "/", segments[-1], res))
         if depth == "1":
-            for name, child in sorted(node.children.items()):
-                if child.is_dir:
-                    resources.append((f"{base}/{encode_path(name)}/", name, child))
-                elif stat.S_ISREG(child.mode):
-                    resources.append((f"{base}/{encode_path(name)}", name, child))
+            for name, child in sorted(self.vfs.children(node), key=lambda kv: kv[0]):
+                child_res = self._resource(child)
+                if child_res.is_dir:
+                    resources.append((f"{base}/{encode_path(name)}/", name, child_res))
+                elif stat.S_ISREG(child_res.mode):
+                    resources.append((f"{base}/{encode_path(name)}", name, child_res))
                 # symlinks and special files are not exposed via WebDAV
         return resources
 
@@ -771,7 +628,8 @@ def _send_dir_listing(self, segments, node, head):
             f' (preserves metadata)">{DOWNLOAD_ICON_SVG}</a>'
         )
         rows = [make_row("../", "..")]
-        children = sorted(node.children.items(), key=lambda kv: (not kv[1].is_dir, kv[0]))
+        children = [(name, self._resource(child)) for name, child in self.vfs.children(node)]
+        children.sort(key=lambda kv: (not kv[1].is_dir, kv[0]))  # directories first
         for name, child in children:
             display_name = remove_surrogates(name)
             if child.is_dir:
@@ -787,8 +645,8 @@ def _send_dir_listing(self, segments, node, head):
                 rows.append(make_row(None, display_name, mtime_ns=child.mtime))
         self._send_page(render_page(title, rows, heading=heading), head)
 
-    def _send_file(self, name, node, pipeline, head):
-        etag = make_etag(node)
+    def _send_file(self, name, node, res, head):
+        etag = make_etag(res)
         if_none_match = self.headers.get("If-None-Match")
         if if_none_match:
             client_tags = [t.strip() for t in if_none_match.split(",")]
@@ -802,78 +660,54 @@ def _send_file(self, name, node, pipeline, head):
         if range_header:
             if_range = self.headers.get("If-Range")
             if if_range is None or if_range.strip() == etag:
-                byte_range = parse_byte_range(range_header, node.size)
+                byte_range = parse_byte_range(range_header, res.size)
         if byte_range == "unsatisfiable":
             self.send_response(416)
-            self.send_header("Content-Range", f"bytes */{node.size}")
+            self.send_header("Content-Range", f"bytes */{res.size}")
             self.send_header("Content-Length", "0")
             self.end_headers()
             return
         if byte_range:
             start, end = byte_range
             self.send_response(206)
-            self.send_header("Content-Range", f"bytes {start}-{end}/{node.size}")
+            self.send_header("Content-Range", f"bytes {start}-{end}/{res.size}")
         else:
-            start, end = 0, node.size - 1
+            start, end = 0, res.size - 1
             self.send_response(200)
         self.send_header("Content-Type", guess_content_type(name))
-        self.send_header("Content-Length", str(end - start + 1 if node.size else 0))
-        self.send_header("Last-Modified", http_date(node.mtime))
+        self.send_header("Content-Length", str(end - start + 1 if res.size else 0))
+        self.send_header("Last-Modified", http_date(res.mtime))
         self.send_header("ETag", etag)
         self.send_header("Accept-Ranges", "bytes")
         self.send_header("X-Content-Type-Options", "nosniff")
         content_disposition = self._content_disposition(name)  # CR/LF-sanitized, see there
         self.send_header("Content-Disposition", content_disposition)
         self.end_headers()
-        if head or node.size == 0:
+        if head or res.size == 0:
             return  # no body to send (and Content-Length is 0 for an empty file)
-        if not node.chunks:
+        chunks = self.vfs.get_item(node.ino).get("chunks")
+        if not chunks:
             # anomaly: a non-empty file with no chunks list (e.g. corrupted metadata). We
             # already sent Content-Length > 0, so abort the connection instead of leaving
             # the client waiting forever for body bytes that will never arrive.
             logger.error(
-                "webdav: file %s has size %d but no chunks, aborting the connection.",
-                remove_surrogates(name),
-                node.size,
+                "webdav: file %s has size %d but no chunks, aborting the connection.", remove_surrogates(name), res.size
             )
             self.close_connection = True
             return
-        # select only the chunks overlapping the requested range, so nothing else
-        # gets fetched and decrypted (the chunk sizes are known in advance).
-        selected, first_offset, pos = [], 0, 0
-        for entry in node.chunks:
-            if pos + entry.size <= start:
-                pos += entry.size
-                continue
-            if pos > end:
-                break
-            if not selected:
-                first_offset = start - pos
-            selected.append(entry)
-            pos += entry.size
-        remaining = end - start + 1
-        for entry in selected:
-            if remaining <= 0:
-                break
-            # serialize repository and data cache access, but write to the client outside
-            # the lock, so one slow client cannot block other requests for the whole download.
-            with self.repo_lock:
-                data = self._get_chunk(pipeline, entry)
-            if data is None:
-                # chunk missing in repository - never serve silently corrupted data:
-                # abort the connection, the client sees a short read (Content-Length mismatch).
-                logger.error(
-                    "webdav: chunk missing while serving %s, aborting the connection.", remove_surrogates(name)
-                )
-                self.close_connection = True
-                return
-            if first_offset:
-                data = data[first_offset:]
-                first_offset = 0
-            if len(data) > remaining:
-                data = data[:remaining]
-            self.wfile.write(data)
-            remaining -= len(data)
+        # the reader fetches only the chunks overlapping the requested range (the chunk
+        # sizes are known in advance), each one under the repository lock - but we write
+        # to the client outside of it, so one slow client cannot block other requests.
+        # pos_key: a mounted file system reads a big file with many sequential range
+        # requests - remember where in the chunk list they got us.
+        try:
+            for data in self.vfs.reader.iter_data(chunks, start, end - start + 1, pos_key=node.ino):
+                self.wfile.write(data)
+        except ChunkMissing:
+            # chunk missing in repository - never serve silently corrupted data:
+            # abort the connection, the client sees a short read (Content-Length mismatch).
+            logger.error("webdav: chunk missing while serving %s, aborting the connection.", remove_surrogates(name))
+            self.close_connection = True
 
     def _send_chunked(self, data):
         """Write one HTTP/1.1 chunked-transfer-encoding block; *data* must be non-empty."""
@@ -895,8 +729,8 @@ def _send_tar(self, segments, head):
 
         The tar size is not known in advance (PAX header sizes vary), so we stream it
         with chunked transfer encoding. Repository access (item iteration and chunk
-        fetching) is serialized under repo_lock, but each chunk is written to the client
-        outside the lock - so a slow client cannot block other requests, and the
+        fetching) is serialized under the VFS lock, but each chunk is written to the
+        client outside the lock - so a slow client cannot block other requests, and the
         LockRefresher can keep the repository lock alive during a long download.
         """
         # lazy import: pulling in the archiver package at module import time would be
@@ -927,16 +761,15 @@ def want(item):
         if head:
             return
 
-        with self.repo_lock:
+        with self.vfs.lock:
             archive = Archive(self.vfs.manifest, self.vfs.archives[archive_name].id)
-            pipeline = archive.pipeline
             item_iter = archive.iter_items(want)
 
         hlm = HardLinkManager(id_type=bytes, info_type=str)  # hlid -> (stripped) path of the first link
         complete = False
         try:
             while True:
-                with self.repo_lock:
+                with self.vfs.lock:
                     try:
                         item = next(item_iter)
                     except StopIteration:
@@ -948,7 +781,7 @@ def want(item):
                     continue  # unsupported item type, skipped (with a warning)
                 tarinfo.pax_headers = item_to_paxheaders("PAX", item)
                 self._send_chunked(tarinfo.tobuf(tarfile.PAX_FORMAT, tarfile.ENCODING, "surrogateescape"))
-                if needs_content and not self._send_tar_content(item, tarinfo.size, pipeline):
+                if needs_content and not self._send_tar_content(item, tarinfo.size):
                     return  # a chunk was missing: leave the chunked stream unterminated (see finally)
             # end-of-archive marker (two zero blocks), then terminate the chunked stream.
             self._send_chunked(b"\0" * (tarfile.BLOCKSIZE * 2))
@@ -960,7 +793,7 @@ def want(item):
                 # tar is truncated (never present a corrupt archive as if it were complete).
                 self.close_connection = True
 
-    def _send_tar_content(self, item, size, pipeline):
+    def _send_tar_content(self, item, size):
         """Stream *item*'s file content into the tar, padded to a 512-byte block boundary.
 
         Returns False (after logging) if the content cannot be produced in full (a chunk
@@ -977,48 +810,21 @@ def _send_tar_content(self, item, size, pipeline):
                 size,
             )
             return False
-        for entry in chunks:
-            # fetch under the lock, write to the client outside it (see _send_tar).
-            with self.repo_lock:
-                data = next(pipeline.fetch_many([entry], ro_type=ROBJ_FILE_STREAM, replacement_chunk=False))
-            if data is None:
-                logger.error(
-                    "webdav: chunk missing while streaming tar member %s, aborting the connection.",
-                    remove_surrogates(item.path),
-                )
-                return False
-            self._send_chunked(data)
+        try:
+            # fetches under the lock, writes to the client outside it (see _send_tar).
+            for data in self.vfs.reader.iter_data(chunks, 0, size):
+                self._send_chunked(data)
+        except ChunkMissing:
+            logger.error(
+                "webdav: chunk missing while streaming tar member %s, aborting the connection.",
+                remove_surrogates(item.path),
+            )
+            return False
         padding = (tarfile.BLOCKSIZE - size % tarfile.BLOCKSIZE) % tarfile.BLOCKSIZE
         if padding:
             self._send_chunked(b"\0" * padding)
         return True
 
-    def _get_chunk(self, pipeline, entry):
-        """Return the decrypted data of chunk *entry*, using the shared data cache.
-
-        Returns None if the chunk is missing in the repository. Must be called while
-        holding repo_lock (it guards both the repository and the data cache, and the
-        data cache's LRUCache is not thread-safe). The cache is keyed by chunk id
-        (a content hash), so it is valid across archives and requests. Reads of a big
-        file over a mounted file system tend to come as many small sequential range
-        requests hitting the same chunk - the cache avoids re-fetching and re-decrypting
-        it every time.
-
-        This complements, but does not duplicate, the repository's own pack cache:
-        fetch_many() -> Repository.get_many() caches whole packs (raw, still encrypted
-        and compressed on-disk bytes), whereas we cache the *decrypted+decompressed*
-        chunk data, i.e. the output of the per-chunk RepoObj.parse() that runs on every
-        fetch even on a pack cache hit. So the two layers save different work (repo I/O
-        vs. per-chunk decrypt+decompress).
-        """
-        try:
-            return self.data_cache[entry.id]
-        except KeyError:
-            data = next(pipeline.fetch_many([entry], ro_type=ROBJ_FILE_STREAM, replacement_chunk=False))
-            if data is not None:
-                self.data_cache[entry.id] = data
-            return data
-
     @staticmethod
     def _content_disposition(name):
         # File names from an archive can contain any byte except NUL and "/", including
@@ -1035,13 +841,6 @@ def _content_disposition(name):
         return strip_crlf(result)  # no-op by construction, but makes the CR/LF safety explicit
 
 
-def data_cache_capacity():
-    # number of decrypted chunks to keep cached; same knob as borg mount uses.
-    # default: number of CPUs (a small, non-zero value), see also borg mount --help.
-    capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1))
-    return max(1, capacity)
-
-
 def make_server(manifest, args, bind="127.0.0.1", port=8000):
     """Create a ThreadingHTTPServer serving the archives selected by *args*.
 
@@ -1049,14 +848,11 @@ def make_server(manifest, args, bind="127.0.0.1", port=8000):
     server threads is serialized with it (use it for e.g. LockRefresher, too).
     """
     repo_lock = threading.RLock()
-    vfs = ArchiveVFS(manifest, args, repo_lock)
-    capacity = data_cache_capacity()
-    logger.debug("webdav: data cache capacity: %d chunks", capacity)
-    data_cache = LRUCache(capacity=capacity)
+    vfs = ArchiveVFS(manifest, args, lock=repo_lock)
+    vfs.create_filesystem()
 
-    handler_class = type("WebDAVHandler", (WebDAVHandler,), dict(vfs=vfs, repo_lock=repo_lock, data_cache=data_cache))
+    handler_class = type("WebDAVHandler", (WebDAVHandler,), dict(vfs=vfs))
     server = ThreadingHTTPServer((bind, port), handler_class)
     server.daemon_threads = True
     server.repo_lock = repo_lock
-    server.data_cache = data_cache
     return server

From cf5276a1cb1146d7b41218e173092abccdc61a00 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 5 Aug 2026 16:50:32 +0200
Subject: [PATCH 024/144] update CHANGES

---
 docs/changes.rst | 99 +++++++++++++++++++++++++++---------------------
 1 file changed, 56 insertions(+), 43 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 629f8315f9..a76c3ccb4b 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -104,7 +104,7 @@ Compatibility notes:
   - removed --numeric-owner (use --numeric-ids)
   - removed --nobsdflags (use --noflags)
   - removed --noatime (default now, see also --atime)
-  - removed --save-space option (does not change behaviour)
+  - removed --save-space option (does not change behavior)
 - removed --bypass-lock option
 - removed --remote-path option (use the BORG_REMOTE_PATH environment variable)
 - removed --rsh option (use the BORG_RSH environment variable)
@@ -169,65 +169,73 @@ New features:
 
 - faster:
 
-  - use multi-threaded zstd compression for big chunks, #9961
-  - use multi-threaded blake3 hashing for big chunks, #9958
-  - extract: avoid refetching/reparsing repeated chunks, #1678
-  - fetch_many: serve all-zero chunks without repository access; also cache
-    recently parsed chunks, #1678
-  - do not verify the chunk id on every read from an encrypted repo (the AEAD
-    authentication covers reads), see BORG_ASSERT_ID, #9994, #7362
+  - create:
+
+    - use multi-threaded zstd compression for big chunks, #9961
+    - use multi-threaded blake3 hashing for big chunks, #9958
+    - overlap pack hash/store and build of the next pack, #9988.
+      BORG_PACK_ASYNC=no disables the store-thread (debugging aid).
+    - chunkers, crypto: release the GIL in pure-C hot paths
+  - extract / mount:
+
+    - avoid refetching/reparsing repeated chunks, #1678
+    - do not verify the chunk id on every read from an encrypted repo (the
+      AEAD authentication covers reads), see BORG_ASSERT_ID, #9994, #7362
+    - serve all-zero chunks without repository access; also cache
+      recently parsed chunks, #1678
 - Chunkers:
 
-  - fastcdc is the new default chunker, for file content data as well as for the
-    item metadata stream, #9957. Compared to the previous default "buzhash", it is
-    faster and its Gear table is derived from secret key material (instead of only
-    XORing a 32bit seed into the table), so chunk cut points are much harder to
-    predict without the key (better resistance against fingerprinting attacks).
-  - rabin-aes: new chunker with cryptographically sound resistance against
-    chunk-size fingerprinting (UHF-then-PRF: secret Rabin polynomial + AES-128,
-    following eprint 2025/558); uses AES hw acceleration or OpenSSL, ~700 MB/s
-  - goldilocks-aes: new chunker, like rabin-aes but with the reference universal
-    hash of eprint 2025/558 (Goldilocks prime-field polynomial hash); about half
-    the rabin-aes speed, mainly a comparison baseline
-  - toeplitz-aes: new chunker, like rabin-aes but with a tabulated LFSR/Toeplitz
-    hash as the universal hash (secret 2 KiB table, fixed public polynomial);
-    optimal 2^-64 collision bound, fastest of the three AES chunkers
-  - fastcdc: SIMD-accelerated scan kernel (NEON on aarch64, AVX2 on x86-64,
-    blocked scalar elsewhere), ~1.4x faster on Apple Silicon; cut points stay
-    bit-identical (1-byte granularity, same golden chunk points)
-  - buzhash64: SIMD scan kernel (same technique and dispatch), ~1.5x faster
-    on Apple Silicon; cut points stay bit-identical
-  - zero-copy fill: the file reader writes data (and zeros for sparse holes)
-    directly into the chunker's scan buffer, replacing several per-byte copy
-    chains; fastcdc ~+40%, buzhash64 ~+20%, buzhash ~+15%,
-    rabin-aes/toeplitz-aes ~+16%, goldilocks-aes ~+9%
-  - lazy buffer compaction: compact the scan buffer only when the free tail
-    gets too small for a full read block (instead of memmoving on every
-    refill), removing ~80% of the compaction copy traffic; cut points stay
-    bit-identical (all CDC chunkers)
+  - fastcdc is the new and faster default chunker, #9957
+  - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034:
+
+    - NEON on aarch64 (e.g. Apple Silicon)
+    - AVX2 on x86-64 (Intel / AMD)
+    - blocked scalar elsewhere
+  - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers
+    (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987
+  - zero-copy fill and lazy buffer compaction optimizations
 - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice
   replacement for `borg mount` in some use cases, #9942
 - mount: expose POSIX ACLs on Linux mounts (not enforced), #1042
 - analyze: report deduplicated size of a set of archives, #5741
+- repo-compress: was temporarily gone, re-added now with pack support, #9663
+- version: add --json output, #10004
+- completion: generate fish completions, remove hand-written ones, #9989
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
 
 Fixes:
 
 - re-add XXH64 to read borg 1.x integrity data, #9935
+- list: add {blake3} format key, #9984
 - support date: archive patterns for --from-borg1, #9949
 - fix calculate_relative_offset year offset for Feb 29, #9967
 - bind pack object header into AEAD authentication
 - fix false repo relocation warning on macOS due to NFC/NFD path differences, #2913
 - release chunk data memoryviews (fixes PyPy memory leak), #1755, #9978
+- fix DownloadPipeline.fetch_many() crashing on a missing chunk, #10024
+- lrucache: make it thread-safe
+- crypto: start a new session after encrypting 2TiB with one aes256-ocb session key, #6501
 
 Other changes:
 
-- crypto: start a new session after encrypting 2TiB with one aes256-ocb session key, #6501
-- chunkers: refactor the shared machinery (buffering, min/max clamping, normalized
-  chunking, iterator protocol) into a common ChunkerBase class instead of keeping a
-  copy per chunker; fastcdc also shares the window-less scan loop with the AES
-  chunkers via a _scan() hook. Cut points stay bit-identical for all chunkers.
+- support Python 3.15
+- borgstore: require 0.6.x, with blake3 support
+- shtab: require >=1.9.3
+- list: validate --format keys, #9984
+- crypto: raise IntegrityError for truncated AEAD/AE envelopes
+- chunkers: refactor the shared machinery into a common ChunkerBase class
+- PackReader.read(): return a memoryview of the in-memory pack instead of copying
+- mount/webdav: unify the 3 archive-as-filesystem implementations, #10020.
+  Behavior changes that fell out of the unification:
+
+  - webdav reads now go through DownloadPipeline.fetch_many(), so the all-zero
+    chunk shortcut and the parsed-chunk cache (#1678) apply to webdav as well.
+  - the mounts get webdav's Unicode NFC lookup fallback (macOS decomposes names).
+  - directories report st_nlink >= 2 (hlfuse behavior) in both mounts.
+  - a chunk that is read to its end is no longer put into the data cache, so a
+    full download does not evict the chunks that partial (range) reads need - this was
+    the FUSE behavior, now webdav shares it.
 - removed some global options (they were difficult to use and spammed the help output):
 
   - --remote-path -> BORG_REMOTE_PATH
@@ -239,11 +247,16 @@ Other changes:
 - docs:
 
   - update README
+  - README: show the contributor chart in the "Helping" section
   - new borg2 demo screencast (see www.borgbackup.org), #6303
   - fix/refactor return codes documentation, #9905
   - fix chunks index / memory usage internals documentation, #9937
   - update help for some commands, #9948
   - fix grammar/typos, #9972
+  - add chunker guide (user-level and cryptographic)
+  - FAME.md: update contributor statistics, #10022
+  - crypto: misc. improvements to code and docs, #6501, ...
+  - GitHub issue #10000: "We Are Borg" joke collection
 
 
 Version 2.0.0b22 (2026-07-22)
@@ -1162,7 +1175,7 @@ New features:
 
 Bug fixes:
 
-- fix Ctrl-C / SIGINT behaviour for pyinstaller-made binaries, #8155
+- fix Ctrl-C / SIGINT behavior for pyinstaller-made binaries, #8155
 - delete: fix error handling with Ctrl-C
 - rcompress: fix error handling with Ctrl-C
 - delete: fix error handling when no archive is specified, #8256
@@ -1192,7 +1205,7 @@ New features:
 - BORG_EXIT_CODES=modern: optional more specific return codes (for errors and warnings).
 
   The default value of this new environment variable is "legacy", which should result in
-  a behaviour similar to borg 1.2 and older (only using rc 0, 1 and 2).
+  a behavior similar to borg 1.2 and older (only using rc 0, 1 and 2).
   "modern" exit codes are much more specific (see the internals/frontends docs).
 - implement "borg version" (shows client and server version), #7829
 
@@ -1441,7 +1454,7 @@ Other changes:
 
 - use local time / local timezone to output timestamps, #7283
 - update development.lock.txt, including a setuptools security fix, #7227
-- remove --save-space option (does not change behaviour)
+- remove --save-space option (does not change behavior)
 - remove part files from final archive
 - remove --consider-part-files, related stats code, update docs
 - transfer: drop part files

From 5e119f60374eb93f6c7c238cadffbabfa9d5566c Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 5 Aug 2026 18:02:29 +0200
Subject: [PATCH 025/144] give each thread its own LZ4 scratch buffer, fixes
 #10032

LZ4 used one module-level Buffer for both directions. Two threads
(de)compressing lz4 chunks concurrently got the same bytearray and wrote
into it at the same time, silently corrupting each other's output - a
blocker for parallel decompression, and lz4 is not an exotic setting.

Thread-locals are the right granularity, not per compressor instance:
compressor instances are shared between threads (LZ4_COMPRESSOR, used by
Auto), a thread's buffer is by definition only used by that thread. The
buffer reuse that avoids a fresh allocation per chunk is kept.
---
 src/borg/compress.pyx               | 29 +++++++++++++++++++++++++--
 src/borg/testsuite/compress_test.py | 31 +++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx
index 739c558cbf..a0d9361810 100644
--- a/src/borg/compress.pyx
+++ b/src/borg/compress.pyx
@@ -20,6 +20,7 @@ import os
 import random
 from struct import Struct
 import sys
+import threading
 import zlib
 
 try:
@@ -86,7 +87,30 @@ cdef extern from "lz4.h":
     int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
     int LZ4_compressBound(int inputSize) nogil
 
-buffer = Buffer(bytearray, size=0)
+_thread_local = threading.local()
+
+
+def get_buffer():
+    """Return the scratch buffer of the calling thread, creating it on first use.
+
+    The LZ4 code below uses a scratch buffer to avoid allocating a new output buffer for
+    every single chunk. Chunks are (de)compressed concurrently by multiple threads, so
+    that buffer must not be shared: two threads getting the same bytearray would write
+    into it at the same time, silently corrupting each other's output.
+
+    Thread-locals are the right granularity here (rather than per compressor instance):
+    compressor instances are shared between threads (e.g. LZ4_COMPRESSOR, used by Auto),
+    while a buffer that belongs to a thread is by definition only used by that thread.
+
+    A thread's buffer stays allocated until the thread ends, and it only ever grows, so
+    a pool of N worker threads may hold N buffers. That is the same behaviour as before,
+    just no longer limited to the one buffer of the main thread.
+    """
+    try:
+        return _thread_local.buffer
+    except AttributeError:
+        buffer = _thread_local.buffer = Buffer(bytearray, size=0)
+        return buffer
 
 cdef class CompressorBase:
     """
@@ -259,7 +283,7 @@ class LZ4(DecidingCompressor):
         cdef char *source = idata
         cdef char *dest
         osize = LZ4_compressBound(isize)
-        buf = buffer.get(osize)
+        buf = get_buffer().get(osize)
         dest = <char *> buf
         with nogil:
             osize = LZ4_compress_default(source, dest, isize, osize)
@@ -283,6 +307,7 @@ class LZ4(DecidingCompressor):
         # a bit more than 8MB is enough for the usual data sizes yielded by the chunker.
         # allocate more if isize * 3 is already bigger, to avoid having to resize often.
         osize = max(int(1.1 * 2**23), isize * 3)
+        buffer = get_buffer()
         while True:
             try:
                 buf = buffer.get(osize)
diff --git a/src/borg/testsuite/compress_test.py b/src/borg/testsuite/compress_test.py
index 6348246f09..35014b8297 100644
--- a/src/borg/testsuite/compress_test.py
+++ b/src/borg/testsuite/compress_test.py
@@ -1,5 +1,7 @@
 import os
+import random
 import zlib
+from concurrent.futures import ThreadPoolExecutor
 
 import pytest
 
@@ -51,6 +53,35 @@ def test_lz4_buffer_allocation(monkeypatch):
     assert incompressible_data == c.decompress(meta, cdata)[1]
 
 
+def test_lz4_threaded():
+    # LZ4 must not share one scratch buffer between threads, see issue #10032.
+    # Without a per-thread buffer, concurrent (de)compression silently produces wrong
+    # output: the threads write their results into the same bytearray.
+    threads = 8
+    rounds = 50
+
+    def make_data(seed):
+        # compressible (so that lz4 does not bail out to "no compression"), but distinct
+        # per thread, so that output of one thread showing up in another one is detected.
+        rnd = random.Random(seed)
+        words = [bytes([rnd.randrange(256)]) * 64 for _ in range(16)]
+        data = bytearray()
+        while len(data) < 2**20:
+            data += rnd.choice(words)
+        return bytes(data)
+
+    def roundtrip(seed):
+        c = Compressor("lz4")
+        data = make_data(seed)
+        for _ in range(rounds):
+            meta, cdata = c.compress({}, data)
+            assert c.decompress(meta, cdata)[1] == data
+
+    with ThreadPoolExecutor(max_workers=threads) as executor:
+        # list() so that exceptions raised in the worker threads are re-raised here
+        list(executor.map(roundtrip, range(threads)))
+
+
 @pytest.mark.parametrize("invalid_cdata", [b"\xff\xfftotalcrap", b"\x08\x00notreallyzlib"])
 def test_autodetect_invalid(invalid_cdata):
     with pytest.raises(ValueError):

From 8ed5cc2fc2f56ec8e89c364775e70ddeefe9d614 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 5 Aug 2026 14:48:09 +0200
Subject: [PATCH 026/144] chunkers: add AVX-512 / VAES scan kernels

fastcdc, buzhash64: AVX-512 variants of the 8-lane candidate test
(one 512-bit vector, vptestnmq fusing the AND and the ==0 test into
a mask register), runtime-detected on x86-64 above the AVX2 kernels.
BORG_FASTCDC_NO_AVX512 / BORG_BUZHASH64_NO_AVX512 cap dispatch at
AVX2 for benchmarking the kernels against each other.

rabin-aes/goldilocks-aes/toeplitz-aes: VAES/AVX-512 variant of the
x86-64 hardware path, kind "vaes": groups of 32 positions encrypted
as 8 zmm vectors of 4 AES blocks each - 4x fewer AES instructions,
register-resident round keys (no per-round reloads), 8 independent
chains to hide the vaesenc latency, vpexpandq digest placement and
a masked vptestnmq cut test without extracts. BORG_PHTE_NO_VAES
caps the AES chunkers at the 128-bit AES-NI path. The VAES path
needs GCC >= 11 / clang >= 14 for __builtin_cpu_supports("vaes");
older compilers keep the AES-NI path.

All kernels return bit-identical cut points; the existing
kernel-identity tests cover the new paths where the CPU has them.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/changes.rst                              |   6 +-
 src/borg/chunkers/buzhash64.pyx               |   2 +-
 src/borg/chunkers/buzhash64_impl.c            |  64 +++++++-
 src/borg/chunkers/buzhash64_impl.h            |   2 +-
 src/borg/chunkers/fastcdc.pyx                 |  10 +-
 src/borg/chunkers/fastcdc_impl.c              |  71 ++++++++-
 src/borg/chunkers/fastcdc_impl.h              |   2 +-
 src/borg/chunkers/goldilocks_aes_impl.h       |   2 +-
 src/borg/chunkers/phte_core.h                 |  35 +++++
 src/borg/chunkers/phte_scan.h                 | 137 +++++++++++++++++-
 src/borg/chunkers/rabin_aes_impl.h            |   2 +-
 src/borg/chunkers/toeplitz_aes_impl.h         |   2 +-
 src/borg/testsuite/chunkers/buzhash64_test.py |   4 +-
 src/borg/testsuite/chunkers/fastcdc_test.py   |   6 +-
 .../testsuite/chunkers/phte_chunkers_test.py  |   2 +-
 15 files changed, 316 insertions(+), 31 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index a76c3ccb4b..7c6a90d785 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -186,13 +186,15 @@ New features:
 - Chunkers:
 
   - fastcdc is the new and faster default chunker, #9957
-  - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034:
+  - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043:
 
     - NEON on aarch64 (e.g. Apple Silicon)
-    - AVX2 on x86-64 (Intel / AMD)
+    - AVX-512 / AVX2 on x86-64 (Intel / AMD)
     - blocked scalar elsewhere
   - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers
     (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987
+  - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64
+    (4 AES blocks per instruction), #10043
   - zero-copy fill and lazy buffer compaction optimizations
 - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice
   replacement for `borg mount` in some use cases, #9942
diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx
index 610e80f83f..4e7b75a40a 100644
--- a/src/borg/chunkers/buzhash64.pyx
+++ b/src/borg/chunkers/buzhash64.pyx
@@ -155,7 +155,7 @@ cdef class ChunkerBuzHash64(ChunkerBase):
 
     @property
     def kernel(self):
-        """Which scan kernel this chunker uses: 'neon', 'avx2', 'blocked' or 'scalar'."""
+        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'."""
         return (<bytes>bz64_kernel_name(self.force_scalar)).decode("ascii")
 
     cdef object process(self):
diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index ca43d60a91..32936e8c3a 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -22,14 +22,15 @@
  * also removes one rotate per byte from the sequential path.
  *
  * Kernel dispatch mirrors fastcdc_impl.c: NEON on aarch64 (baseline there),
- * AVX2 on x86-64 (runtime-detected), blocked scalar elsewhere, sequential
- * when forced. Measured on Apple M-series (64 MiB, window 4095, 21-bit
+ * AVX-512 or AVX2 on x86-64 (runtime-detected), blocked scalar elsewhere,
+ * sequential when forced. Measured on Apple M-series (64 MiB, window 4095, 21-bit
  * mask): sequential 1150, blocked scalar 2340, NEON 2260 MB/s - NEON and
  * blocked are a tie here (throughput and energy, within noise), because the
  * XOR/AND/compare test also runs well on the scalar ALUs; the symmetric
  * dispatch is kept for consistency across the SIMD chunker kernels.
  * All kernels return bit-identical results. */
 
+#include <stdlib.h>
 #include <string.h>
 
 #include "buzhash64_impl.h"
@@ -209,8 +210,56 @@ bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
+/* --- AVX-512 (x86-64, runtime detected) ---------------------------------- */
+
+#define BZ_KIND_512 "avx512"
+
+/* The same 8-lane test as the AVX2 kernel in one 512-bit vector: vptestnmq
+ * fuses the AND and the ==0 test per lane into a mask register. Only the
+ * test narrows; the scalar prefix work (bz64_block_prefix) is shared. */
+__attribute__((target("avx512f"))) static size_t
+bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot,
+                  const uint8_t *pr, const uint8_t *pa,
+                  size_t n, uint64_t *sum_io, uint64_t mask)
+{
+    uint64_t sum = *sum_io;
+    uint64_t s[8];
+    size_t j = 0;
+    /* _mm512_set_epi64 takes arguments high lane first: lane k = ROTL(mask, 7-k) */
+    __m512i M = _mm512_set_epi64((long long)mask, (long long)BZ_ROTL(mask, 1),
+                                 (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3),
+                                 (long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5),
+                                 (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7));
+
+    while (j + 8 <= n && (sum & mask) != 0) {
+        bz64_block_prefix(T, Trot, pr + j, pa + j, s);
+        uint64_t c = BZ_ROTL(sum, 8);
+        __m512i H = _mm512_xor_si512(_mm512_set1_epi64((long long)c),
+                                     _mm512_loadu_si512((const void *)s));
+        if (_mm512_testn_epi64_mask(H, M)) {
+            size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */
+            *sum_io = sum;
+            return j + r;
+        }
+        sum = c ^ s[7];
+        j += 8;
+    }
+    if (j < n)
+        j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask);
+    *sum_io = sum;
+    return j;
+}
+
 static int bz64_simd_available(void)
 {
+    /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2 (for benchmarking the
+     * kernels against each other); like the detection itself it is read
+     * once per process, so it must be set before the first chunker use. */
+    if (__builtin_cpu_supports("avx512f")) {
+        const char *e = getenv("BORG_BUZHASH64_NO_AVX512");
+        if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0)
+            return 2;
+    }
     return __builtin_cpu_supports("avx2");
 }
 
@@ -233,7 +282,8 @@ static int bz64_simd_available(void)
 
 /* --- dispatch ----------------------------------------------------------- */
 
-/* resolved once; a racy double-init writes the same value, so it is benign */
+/* resolved once (0 = blocked, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined);
+ * a racy double-init writes the same value, so it is benign */
 static int bz64_use_simd = -1;
 
 size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
@@ -244,6 +294,10 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
         return bz64_scan_seq(table, table_rot, p_rem, p_add, n, sum, mask);
     if (bz64_use_simd < 0)
         bz64_use_simd = bz64_simd_available();
+#ifdef BZ_KIND_512
+    if (bz64_use_simd == 2)
+        return bz64_scan_simd512(table, table_rot, p_rem, p_add, n, sum, mask);
+#endif
     if (bz64_use_simd)
         return bz64_scan_simd(table, table_rot, p_rem, p_add, n, sum, mask);
     return bz64_scan_blocked(table, table_rot, p_rem, p_add, n, sum, mask);
@@ -255,5 +309,9 @@ const char *bz64_kernel_name(int force_scalar)
         return "scalar";
     if (bz64_use_simd < 0)
         bz64_use_simd = bz64_simd_available();
+#ifdef BZ_KIND_512
+    if (bz64_use_simd == 2)
+        return BZ_KIND_512;
+#endif
     return bz64_use_simd ? BZ_KIND : "blocked";
 }
diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h
index cd805341c7..2a9220cc73 100644
--- a/src/borg/chunkers/buzhash64_impl.h
+++ b/src/borg/chunkers/buzhash64_impl.h
@@ -27,7 +27,7 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                  const uint8_t *p_rem, const uint8_t *p_add,
                  size_t n, uint64_t *sum, uint64_t mask, int force_scalar);
 
-/* Name of the kernel bz64_scan would use: "neon", "avx2", "blocked" or "scalar". */
+/* Name of the kernel bz64_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */
 const char *bz64_kernel_name(int force_scalar);
 
 #endif
diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx
index 7172fe94fc..535f189896 100644
--- a/src/borg/chunkers/fastcdc.pyx
+++ b/src/borg/chunkers/fastcdc.pyx
@@ -33,9 +33,11 @@ cdef extern from "fastcdc_impl.h":
 # class only provides the keyed Gear table and the _scan() hook calling the C kernel.
 #
 # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on
-# aarch64, AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to the plain
-# sequential Gear loop: every byte position is tested, cuts and hash state are exactly the
-# same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop (for tests).
+# aarch64, AVX-512 or AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to
+# the plain sequential Gear loop: every byte position is tested, cuts and hash state are
+# exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop
+# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 (for benchmarking, must be
+# set before the first chunker use).
 
 
 @cython.boundscheck(False)
@@ -85,7 +87,7 @@ cdef class ChunkerFastCDC(ChunkerBase):
 
     @property
     def kernel(self):
-        """Which scan kernel this chunker uses: 'neon', 'avx2', 'blocked' or 'scalar'."""
+        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'."""
         return (<bytes>fc_kernel_name(self.force_scalar)).decode("ascii")
 
     cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept:
diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c
index f7c184caa3..4ddfbb1d82 100644
--- a/src/borg/chunkers/fastcdc_impl.c
+++ b/src/borg/chunkers/fastcdc_impl.c
@@ -21,9 +21,11 @@
  * which an exact sequential recheck of the block resolves. The per-lane
  * masks (mask << 7 .. mask << 0) are constants of the scan.
  *
- * All kernels (sequential, blocked scalar, NEON, AVX2) return bit-identical
- * cut positions and fp values; the chunker's golden tests depend on this. */
+ * All kernels (sequential, blocked scalar, NEON, AVX2, AVX-512) return
+ * bit-identical cut positions and fp values; the chunker's golden tests
+ * depend on this. */
 
+#include <stdlib.h>
 #include <string.h>
 
 #include "fastcdc_impl.h"
@@ -214,8 +216,62 @@ fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io,
     return -1;
 }
 
+/* --- AVX-512 (x86-64, runtime detected) --------------------------------- */
+
+#define FC_KIND_512 "avx512"
+
+/* The same 8-lane candidate test as the AVX2 kernel, in one 512-bit vector:
+ * vptestnmq fuses the AND and the ==0 test per lane into a mask register,
+ * so the whole test is broadcast + load + add + testn + branch. Only the
+ * test narrows; the scalar prefix work (fc_block_prefix) is shared. */
+__attribute__((target("avx512f"))) static int64_t
+fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
+{
+    uint64_t fp = *fp_io;
+    uint64_t s[8];
+    /* _mm512_set_epi64 takes arguments high lane first: lane j = mask << (7-j) */
+    __m512i M = _mm512_set_epi64((long long)mask, (long long)(mask << 1),
+                                 (long long)(mask << 2), (long long)(mask << 3),
+                                 (long long)(mask << 4), (long long)(mask << 5),
+                                 (long long)(mask << 6), (long long)(mask << 7));
+    size_t i = 0;
+
+    for (; i + 8 <= n; i += 8) {
+        fc_block_prefix(gear, p + i, s);
+        uint64_t c = fp << 8;
+        __m512i H = _mm512_add_epi64(_mm512_set1_epi64((long long)c),
+                                     _mm512_loadu_si512((const void *)s));
+        if (_mm512_testn_epi64_mask(H, M)) {
+            int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */
+            if (r >= 0) {
+                *fp_io = fp;
+                return (int64_t)i + r;
+            }
+        } else {
+            fp = c + s[7];
+        }
+    }
+    if (i < n) {
+        int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask);
+        if (r >= 0) {
+            *fp_io = fp;
+            return (int64_t)i + r;
+        }
+    }
+    *fp_io = fp;
+    return -1;
+}
+
 static int fc_simd_available(void)
 {
+    /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2 (for benchmarking the
+     * kernels against each other); like the detection itself it is read
+     * once per process, so it must be set before the first chunker use. */
+    if (__builtin_cpu_supports("avx512f")) {
+        const char *e = getenv("BORG_FASTCDC_NO_AVX512");
+        if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0)
+            return 2;
+    }
     return __builtin_cpu_supports("avx2");
 }
 
@@ -236,7 +292,8 @@ static int fc_simd_available(void)
 
 /* --- dispatch ----------------------------------------------------------- */
 
-/* resolved once; a racy double-init writes the same value, so it is benign */
+/* resolved once (0 = blocked, 1 = FC_KIND, 2 = FC_KIND_512 where defined);
+ * a racy double-init writes the same value, so it is benign */
 static int fc_use_simd = -1;
 
 int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar)
@@ -245,6 +302,10 @@ int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp,
         return fc_scan_seq(gear, p, n, fp, mask);
     if (fc_use_simd < 0)
         fc_use_simd = fc_simd_available();
+#ifdef FC_KIND_512
+    if (fc_use_simd == 2)
+        return fc_scan_simd512(gear, p, n, fp, mask);
+#endif
     if (fc_use_simd)
         return fc_scan_simd(gear, p, n, fp, mask);
     return fc_scan_blocked(gear, p, n, fp, mask);
@@ -256,5 +317,9 @@ const char *fc_kernel_name(int force_scalar)
         return "scalar";
     if (fc_use_simd < 0)
         fc_use_simd = fc_simd_available();
+#ifdef FC_KIND_512
+    if (fc_use_simd == 2)
+        return FC_KIND_512;
+#endif
     return fc_use_simd ? FC_KIND : "blocked";
 }
diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h
index 10887ec3d8..a66606c707 100644
--- a/src/borg/chunkers/fastcdc_impl.h
+++ b/src/borg/chunkers/fastcdc_impl.h
@@ -20,7 +20,7 @@
  * All kernels return bit-identical results. */
 int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar);
 
-/* Name of the kernel fc_scan would use: "neon", "avx2", "blocked" or "scalar". */
+/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */
 const char *fc_kernel_name(int force_scalar);
 
 #endif
diff --git a/src/borg/chunkers/goldilocks_aes_impl.h b/src/borg/chunkers/goldilocks_aes_impl.h
index 0b0dd148ab..e758cae44a 100644
--- a/src/borg/chunkers/goldilocks_aes_impl.h
+++ b/src/borg/chunkers/goldilocks_aes_impl.h
@@ -43,7 +43,7 @@ GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t a
 
 void gl_free(GL_CTX *ctx);
 
-/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */
+/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */
 const char *gl_kind(const GL_CTX *ctx);
 
 /* Full (non-rolling) polynomial hash of the 64 bytes at q (Horner): the
diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h
index fe9856e6b1..ec8ad6a2e4 100644
--- a/src/borg/chunkers/phte_core.h
+++ b/src/borg/chunkers/phte_core.h
@@ -15,6 +15,7 @@
 
 #include <stdint.h>
 #include <stddef.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include <openssl/evp.h>
@@ -25,6 +26,7 @@ typedef struct {
     uint8_t rk[11][16];  /* AES-128 round keys, for the hardware paths */
     EVP_CIPHER_CTX *evp; /* portable path */
     int use_hw;
+    int use_hw512; /* VAES/AVX-512 variant of the hardware path (x86-64) */
 } PHTE_BASE;
 
 /* --- endianness-explicit helpers (byte loops, so cut points do not depend
@@ -130,10 +132,34 @@ phte_aes1_ni(const uint8_t rk[11][16], __m128i b)
     return _mm_aesenclast_si128(b, _mm_loadu_si128((const __m128i *)rk[10]));
 }
 
+/* VAES/AVX-512 variant of the hardware path (4 AES blocks per instruction).
+ * __builtin_cpu_supports("vaes") needs GCC >= 11 / clang >= 14; older
+ * compilers simply keep the 128-bit AES-NI path. */
+#if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 11) || (defined(__clang__) && __clang_major__ >= 14)
+#define PHTE_HAVE_HW512 1
+#define PHTE_KIND_HW512 "vaes"
+
+static int phte_hw512_available(void)
+{
+    /* BORG_PHTE_NO_VAES caps the AES chunkers at the 128-bit AES-NI path
+     * (for benchmarking the paths against each other); it is read at
+     * context creation, ""/"0" mean off. */
+    const char *e;
+    if (!__builtin_cpu_supports("avx512f") || !__builtin_cpu_supports("vaes"))
+        return 0;
+    e = getenv("BORG_PHTE_NO_VAES");
+    return e == NULL || e[0] == 0 || strcmp(e, "0") == 0;
+}
+#endif
+
 #else
 #define PHTE_HAVE_HW 0
 #endif
 
+#ifndef PHTE_HAVE_HW512
+#define PHTE_HAVE_HW512 0
+#endif
+
 /* --- context base management ------------------------------------------- */
 
 /* Expand the AES key, select the scan path and set up the OpenSSL context.
@@ -146,6 +172,11 @@ static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int force_sw)
 #else
     (void)force_sw;
     b->use_hw = 0;
+#endif
+#if PHTE_HAVE_HW512
+    b->use_hw512 = b->use_hw && phte_hw512_available();
+#else
+    b->use_hw512 = 0;
 #endif
     b->evp = EVP_CIPHER_CTX_new();
     if (b->evp == NULL)
@@ -169,6 +200,10 @@ static void phte_base_free(PHTE_BASE *b)
 
 static const char *phte_base_kind(const PHTE_BASE *b)
 {
+#if PHTE_HAVE_HW512
+    if (b->use_hw512)
+        return PHTE_KIND_HW512;
+#endif
 #if PHTE_HAVE_HW
     if (b->use_hw)
         return PHTE_KIND_HW;
diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h
index 4695cae788..776b2360da 100644
--- a/src/borg/chunkers/phte_scan.h
+++ b/src/borg/chunkers/phte_scan.h
@@ -19,9 +19,11 @@
  *  - a portable path batching digests through OpenSSL EVP AES-128-ECB,
  *  - a hardware path using AES instructions directly (arm64 crypto extension,
  *    x86-64 AES-NI), which interleaves the serial rolling-hash chain with
- *    pipelined AES so the AES work is (mostly) hidden behind it.
+ *    pipelined AES so the AES work is (mostly) hidden behind it,
+ *  - a VAES/AVX-512 variant of the x86-64 hardware path (runtime-detected)
+ *    encrypting 4 AES blocks per instruction.
  *
- * Both paths produce bit-identical cut points.
+ * All paths produce bit-identical cut points.
  */
 
 #define PH_CAT2(a, b) a##b
@@ -105,11 +107,12 @@ static int64_t PH_FN(scan_evp)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *
 
 /* --- hardware paths ----------------------------------------------------
  *
- * Both hardware paths process groups of 8 positions: the two rolling lanes
- * advance 4 stride-2 steps each (two independent dependency chains), then the
- * 8 digests are encrypted with interleaved AES instructions, which execute on
- * different ports than the rolling work and thus overlap with the next
- * group's.
+ * The 128-bit hardware paths process groups of 8 positions (the VAES/AVX-512
+ * path below uses the same structure with groups of 32): the two rolling
+ * lanes advance 4 stride-2 steps each (two independent dependency chains),
+ * then the 8 digests are encrypted with interleaved AES instructions, which
+ * execute on different ports than the rolling work and thus overlap with the
+ * next group's.
  *
  * Loop invariant: entering a group at position i, db = digest at i and
  * da = digest at i+1 (both already computed); the group emits digests for
@@ -334,10 +337,130 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t
 
 #endif /* PHTE_HAVE_HW */
 
+#if PHTE_HAVE_HW512
+
+/* VAES/AVX-512 variant of the x86-64 hardware path: the same two rolling
+ * lanes, but groups of 32 positions whose digests are encrypted as 8 zmm
+ * vectors of 4 AES blocks each - 4x fewer AES instructions than the 128-bit
+ * path, the round keys stay register-resident (32 zmm registers instead of
+ * 16 xmm, so no per-round key reloads), and the 8 independent chains hide
+ * the vaesenc latency (2 chains would be latency-bound). vpexpandq places a
+ * vector's 4 digests into its block-low qwords in one instruction, and a
+ * masked vptestnmq tests the 4 ciphertext-low qwords against the mask
+ * without extracting. Every digest is exact and every position is tested,
+ * so cut points stay bit-identical to the other paths. */
+__attribute__((target("aes,sse2,vaes,avx512f"))) static int64_t
+PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask)
+{
+    __m512i k[11];
+    __m512i M = _mm512_set1_epi64((long long)mask);
+    uint64_t d = *digest, da, db;
+    const uint8_t *qo = p - 64;
+    size_t i = 0;
+    int lanes_live = 0;
+
+    /* broadcast each 128-bit round key to all four block lanes */
+    for (int j = 0; j < 11; j++)
+        k[j] = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c->base.rk[j]));
+
+    if (n >= 34) {
+        db = PH_ROLL(c, d, qo[0], p[0]);               /* d_0 */
+        da = PH_ROLL2(c, d, qo[0], qo[1], p[0], p[1]); /* d_1 */
+        lanes_live = 1;
+    }
+
+    while (lanes_live && i + 34 <= n) {
+        uint64_t dg[32];
+        __m512i b0, b1, b2, b3, b4, b5, b6, b7;
+        __mmask8 h0, h1, h2, h3, h4, h5, h6, h7;
+
+        dg[0] = db;
+        dg[1] = da;
+        for (int j = 2; j < 32; j += 2) {
+            db = PH_ROLL2(c, db, qo[i + j - 1], qo[i + j], p[i + j - 1], p[i + j]);
+            dg[j] = db;
+            da = PH_ROLL2(c, da, qo[i + j], qo[i + j + 1], p[i + j], p[i + j + 1]);
+            dg[j + 1] = da;
+        }
+        /* prepare the next group's invariant (digests at i+32, i+33) */
+        db = PH_ROLL2(c, db, qo[i + 31], qo[i + 32], p[i + 31], p[i + 32]);
+        da = PH_ROLL2(c, da, qo[i + 32], qo[i + 33], p[i + 32], p[i + 33]);
+
+        /* qword lanes 0,2,4,6 take 4 digests, lanes 1,3,5,7 stay zero
+         * (= AES input block bytes 8..15, as in the other paths) */
+        b0 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg), k[0]);
+        b1 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 4), k[0]);
+        b2 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 8), k[0]);
+        b3 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 12), k[0]);
+        b4 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 16), k[0]);
+        b5 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 20), k[0]);
+        b6 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 24), k[0]);
+        b7 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 28), k[0]);
+        for (int r = 1; r < 10; r++) {
+            b0 = _mm512_aesenc_epi128(b0, k[r]);
+            b1 = _mm512_aesenc_epi128(b1, k[r]);
+            b2 = _mm512_aesenc_epi128(b2, k[r]);
+            b3 = _mm512_aesenc_epi128(b3, k[r]);
+            b4 = _mm512_aesenc_epi128(b4, k[r]);
+            b5 = _mm512_aesenc_epi128(b5, k[r]);
+            b6 = _mm512_aesenc_epi128(b6, k[r]);
+            b7 = _mm512_aesenc_epi128(b7, k[r]);
+        }
+        h0 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b0, k[10]), M);
+        h1 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b1, k[10]), M);
+        h2 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b2, k[10]), M);
+        h3 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b3, k[10]), M);
+        h4 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b4, k[10]), M);
+        h5 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b5, k[10]), M);
+        h6 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b6, k[10]), M);
+        h7 = _mm512_mask_testn_epi64_mask(0x55, _mm512_aesenclast_epi128(b7, k[10]), M);
+
+        if (h0 | h1 | h2 | h3 | h4 | h5 | h6 | h7) {
+            /* block j of vector g is qword lane 2j and position i + 4g + j */
+            const uint8_t hs[8] = {h0, h1, h2, h3, h4, h5, h6, h7};
+            for (int j = 0; j < 32; j++) {
+                if ((hs[j >> 2] >> ((j & 3) * 2)) & 1) {
+                    *digest = dg[j];
+                    return (int64_t)(i + j);
+                }
+            }
+        }
+        i += 32;
+    }
+    /* tail: single positions, exactly as in the 128-bit path */
+    if (lanes_live && i < n) {
+        __m128i b = phte_aes1_ni(c->base.rk, _mm_set_epi64x(0, (long long)db));
+        if (((uint64_t)_mm_cvtsi128_si64(b) & mask) == 0) {
+            *digest = db;
+            return (int64_t)i;
+        }
+        d = db;
+        i++;
+    }
+    while (i < n) {
+        __m128i b;
+        d = PH_ROLL(c, d, qo[i], p[i]);
+        b = phte_aes1_ni(c->base.rk, _mm_set_epi64x(0, (long long)d));
+        if (((uint64_t)_mm_cvtsi128_si64(b) & mask) == 0) {
+            *digest = d;
+            return (int64_t)i;
+        }
+        i++;
+    }
+    *digest = d;
+    return -1;
+}
+
+#endif /* PHTE_HAVE_HW512 */
+
 /* --- dispatch (exported) ------------------------------------------------ */
 
 int64_t PH_FN(scan)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask)
 {
+#if PHTE_HAVE_HW512
+    if (c->base.use_hw512)
+        return PH_FN(scan_hw512)(c, p, n, digest, mask);
+#endif
 #if PHTE_HAVE_HW
     if (c->base.use_hw)
         return PH_FN(scan_hw)(c, p, n, digest, mask);
diff --git a/src/borg/chunkers/rabin_aes_impl.h b/src/borg/chunkers/rabin_aes_impl.h
index 655c8b206b..22fe17ad46 100644
--- a/src/borg/chunkers/rabin_aes_impl.h
+++ b/src/borg/chunkers/rabin_aes_impl.h
@@ -40,7 +40,7 @@ RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw);
 
 void ra_free(RA_CTX *ctx);
 
-/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */
+/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */
 const char *ra_kind(const RA_CTX *ctx);
 
 /* Full (non-rolling) Rabin digest of the 64 bytes at q: the window warm-up at
diff --git a/src/borg/chunkers/toeplitz_aes_impl.h b/src/borg/chunkers/toeplitz_aes_impl.h
index bc005dcf5b..dec05d32e7 100644
--- a/src/borg/chunkers/toeplitz_aes_impl.h
+++ b/src/borg/chunkers/toeplitz_aes_impl.h
@@ -44,7 +44,7 @@ TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw);
 
 void tp_free(TP_CTX *ctx);
 
-/* Which path this context uses: "aes-arm64", "aes-ni" or "evp". */
+/* Which path this context uses: "aes-arm64", "vaes", "aes-ni" or "evp". */
 const char *tp_kind(const TP_CTX *ctx);
 
 /* Full (non-rolling) digest of the 64 bytes at q: the window warm-up at the
diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py
index 452a5b0583..ebb9849475 100644
--- a/src/borg/testsuite/chunkers/buzhash64_test.py
+++ b/src/borg/testsuite/chunkers/buzhash64_test.py
@@ -143,7 +143,7 @@ def rnd_key():
 
 
 def test_buzhash64_kernels_identical():
-    # the blocked/AVX2 scan kernel (auto-selected) and the plain sequential
+    # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the plain sequential
     # loop must produce identical cut points.
     data = os.urandom(4 * 1024 * 1024)
     key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
@@ -162,4 +162,4 @@ def sizes(chunker):
         del os.environ["BORG_BUZHASH64_FORCE_SCALAR"]
     assert sizes_default == sizes_scalar
     # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("neon", "avx2", "blocked", "scalar")
+    assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar")
diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py
index 68812f64ca..d2ce69f3e0 100644
--- a/src/borg/testsuite/chunkers/fastcdc_test.py
+++ b/src/borg/testsuite/chunkers/fastcdc_test.py
@@ -179,8 +179,8 @@ def rnd_key():
 
 
 def test_fastcdc_kernels_identical():
-    # the SIMD scan kernel (neon/avx2/blocked, auto-selected) and the plain
-    # sequential Gear loop must produce identical cut points.
+    # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the
+    # plain sequential Gear loop must produce identical cut points.
     data = os.urandom(4 * 1024 * 1024)
     key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
 
@@ -198,4 +198,4 @@ def sizes(chunker):
         del os.environ["BORG_FASTCDC_FORCE_SCALAR"]
     assert sizes_default == sizes_scalar
     # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("neon", "avx2", "blocked", "scalar")
+    assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar")
diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py
index 2ef3f0c219..2dc2cac3e6 100644
--- a/src/borg/testsuite/chunkers/phte_chunkers_test.py
+++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py
@@ -61,7 +61,7 @@ def sizes(chunker):
         del os.environ[env_var]
     assert sizes_default == sizes_evp
     # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("aes-arm64", "aes-ni", "evp")
+    assert default.kernel in ("aes-arm64", "vaes", "aes-ni", "evp")
 
 
 def test_chunksize_distribution(chunker_spec):

From 17ded35f186b389d4f34409882b0cfb778a01176 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 5 Aug 2026 14:52:53 +0200
Subject: [PATCH 027/144] chunkers: add BORG_FASTCDC_NO_AVX2 /
 BORG_BUZHASH64_NO_AVX2

Cap dispatch at the blocked scalar kernel, completing the bench
ladder on x86-64: default (avx512) -> NO_AVX512 (avx2) -> NO_AVX2
(blocked) -> FORCE_SCALAR (sequential). Read once per process, like
the CPU detection itself.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/chunkers/buzhash64_impl.c | 22 ++++++++++++++--------
 src/borg/chunkers/fastcdc.pyx      |  4 ++--
 src/borg/chunkers/fastcdc_impl.c   | 22 ++++++++++++++--------
 3 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index 32936e8c3a..5e8a3568ee 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -250,16 +250,22 @@ bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
+static int bz64_env_set(const char *name)
+{
+    const char *e = getenv(name);
+    return e != NULL && e[0] != 0 && strcmp(e, "0") != 0;
+}
+
 static int bz64_simd_available(void)
 {
-    /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2 (for benchmarking the
-     * kernels against each other); like the detection itself it is read
-     * once per process, so it must be set before the first chunker use. */
-    if (__builtin_cpu_supports("avx512f")) {
-        const char *e = getenv("BORG_BUZHASH64_NO_AVX512");
-        if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0)
-            return 2;
-    }
+    /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2, BORG_BUZHASH64_NO_AVX2
+     * at the blocked scalar kernel (for benchmarking the kernels against
+     * each other); like the detection itself they are read once per process,
+     * so they must be set before the first chunker use. */
+    if (bz64_env_set("BORG_BUZHASH64_NO_AVX2"))
+        return 0;
+    if (__builtin_cpu_supports("avx512f") && !bz64_env_set("BORG_BUZHASH64_NO_AVX512"))
+        return 2;
     return __builtin_cpu_supports("avx2");
 }
 
diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx
index 535f189896..8d915d59c1 100644
--- a/src/borg/chunkers/fastcdc.pyx
+++ b/src/borg/chunkers/fastcdc.pyx
@@ -36,8 +36,8 @@ cdef extern from "fastcdc_impl.h":
 # aarch64, AVX-512 or AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to
 # the plain sequential Gear loop: every byte position is tested, cuts and hash state are
 # exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop
-# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 (for benchmarking, must be
-# set before the first chunker use).
+# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 and BORG_FASTCDC_NO_AVX2=1
+# at the blocked scalar kernel (for benchmarking, must be set before the first chunker use).
 
 
 @cython.boundscheck(False)
diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c
index 4ddfbb1d82..46dfd141c6 100644
--- a/src/borg/chunkers/fastcdc_impl.c
+++ b/src/borg/chunkers/fastcdc_impl.c
@@ -262,16 +262,22 @@ fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_i
     return -1;
 }
 
+static int fc_env_set(const char *name)
+{
+    const char *e = getenv(name);
+    return e != NULL && e[0] != 0 && strcmp(e, "0") != 0;
+}
+
 static int fc_simd_available(void)
 {
-    /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2 (for benchmarking the
-     * kernels against each other); like the detection itself it is read
-     * once per process, so it must be set before the first chunker use. */
-    if (__builtin_cpu_supports("avx512f")) {
-        const char *e = getenv("BORG_FASTCDC_NO_AVX512");
-        if (e == NULL || e[0] == 0 || strcmp(e, "0") == 0)
-            return 2;
-    }
+    /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2, BORG_FASTCDC_NO_AVX2 at
+     * the blocked scalar kernel (for benchmarking the kernels against each
+     * other); like the detection itself they are read once per process, so
+     * they must be set before the first chunker use. */
+    if (fc_env_set("BORG_FASTCDC_NO_AVX2"))
+        return 0;
+    if (__builtin_cpu_supports("avx512f") && !fc_env_set("BORG_FASTCDC_NO_AVX512"))
+        return 2;
     return __builtin_cpu_supports("avx2");
 }
 

From ddd54be9f24daa45e9fb7ce4e283d2038cccb3f0 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 01:13:59 +0200
Subject: [PATCH 028/144] chunkers: build the aligned domain in the vector
 kernels

The AVX2 and AVX-512 scan kernels of fastcdc and buzhash64 only vectorized
the 8-lane candidate test: the per-block prefix work stayed scalar and was
handed over through the stack. Eight 8-byte stores immediately followed by
one 32/64-byte vector load cannot use store-to-load forwarding, so every
block paid a forwarding stall - enough that the fastcdc AVX2 kernel was
slower than the blocked scalar fallback it is supposed to beat.

Both vector kernels now:

- do the table lookups for the next block while the current one is tested
  (double buffer), which puts a full loop body between the stores and the
  load, so the stores have retired by then and the stall disappears, and
- build the aligned domain in the vector domain: vpsllvq / vprolvq for the
  per-lane shifts resp. rotations, then the prefix sum / prefix XOR over the
  lanes - three valignq steps on AVX-512, two vpermq steps per 4-lane half
  plus a carry on AVX2. Only the table lookups stay scalar.

The serial chain across blocks stays what it was (one scalar add + shift
resp. rotate); s[7] is read out of the vector with vpermq off that chain.

A gather-based kernel avoiding the round-trip entirely was tried and
dropped: vpgatherqq is too slow on Zen 4 (1102 vs 2778 MB/s). Pipelining
two blocks ahead instead of one also regressed.

All kernels still return bit-identical cut points; the kernel-identity and
fuzz tests cover this, and chunk boundaries were verified equal across the
avx512, avx2 and blocked tiers for all chunkers.

Measured on an AMD Ryzen 5 8500GE (Zen 4/4c, AVX-512 on a 256-bit datapath),
borg benchmark cpu, 1 GB, pinned:

    fastcdc     0.818s -> 0.381s  (2.15x)
    buzhash64   0.906s -> 0.667s  (1.36x)

Per kernel tier (MB/s, isolated chunker benchmark):

    fastcdc    avx512  1173 -> 2689     avx2   800 -> 2583
    buzhash64  avx512  1052 -> 1509     avx2  1026 -> 1478

AVX-512 stays close to AVX2 here because Zen 4 double-pumps 512-bit ops;
the win is the removed stall and the vectorized prefix, not the width.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/buzhash64_impl.c | 167 +++++++++++++++++++++--------
 src/borg/chunkers/fastcdc_impl.c   | 163 ++++++++++++++++++++--------
 2 files changed, 239 insertions(+), 91 deletions(-)

diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index 5e8a3568ee..e54f45374b 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -172,39 +172,99 @@ static int bz64_simd_available(void)
 
 #include <immintrin.h>
 
+/* Load the block's 8 per-byte deltas D_t = Trot[out_t] ^ T[in_t] (via two
+ * 8-byte data loads), without the rotations and the prefix XOR: the vector
+ * kernels do those in the vector domain. Endianness-independent. */
+static inline void bz64_block_delta(const uint64_t *T, const uint64_t *Trot,
+                                    const uint8_t *pr, const uint8_t *pa, uint64_t d[8])
+{
+    uint64_t wr, wa;
+    memcpy(&wr, pr, 8);
+    memcpy(&wa, pa, 8);
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    wr = __builtin_bswap64(wr);
+    wa = __builtin_bswap64(wa);
+#endif
+    d[0] = Trot[(uint8_t)wr] ^ T[(uint8_t)wa];
+    d[1] = Trot[(uint8_t)(wr >> 8)] ^ T[(uint8_t)(wa >> 8)];
+    d[2] = Trot[(uint8_t)(wr >> 16)] ^ T[(uint8_t)(wa >> 16)];
+    d[3] = Trot[(uint8_t)(wr >> 24)] ^ T[(uint8_t)(wa >> 24)];
+    d[4] = Trot[(uint8_t)(wr >> 32)] ^ T[(uint8_t)(wa >> 32)];
+    d[5] = Trot[(uint8_t)(wr >> 40)] ^ T[(uint8_t)(wa >> 40)];
+    d[6] = Trot[(uint8_t)(wr >> 48)] ^ T[(uint8_t)(wa >> 48)];
+    d[7] = Trot[(uint8_t)(wr >> 56)] ^ T[(uint8_t)(wa >> 56)];
+}
+
+/* AVX2 has no variable rotate, so ROTL(v, k) is built from a variable left
+ * and right shift. The k = 0 lane needs a 64-bit right shift, which
+ * vpsrlvq defines as zero - exactly what (v << 0) | 0 = v needs. */
+__attribute__((target("avx2"))) static inline __m256i bz64_rotl4(__m256i v, __m256i k, __m256i kc)
+{
+    return _mm256_or_si256(_mm256_sllv_epi64(v, k), _mm256_srlv_epi64(v, kc));
+}
+
+/* The 8-lane test in two 256-bit vectors, with the aligned domain built in
+ * the vector domain: per-lane rotations, then a prefix XOR inside each 4-lane
+ * half (two vpermq+vpxor steps) plus a broadcast of the low half's total into
+ * the high half.
+ *
+ * Only the 2x8 table lookups stay scalar, and they are done one block ahead
+ * into a double buffer: a 32-byte vector load placed right after the eight
+ * 8-byte scalar stores that filled it cannot use store-to-load forwarding and
+ * stalls, twice per block. Loading block j+1 while block j is tested puts a
+ * full loop body between the stores and the load, so the stall disappears. */
 __attribute__((target("avx2"))) static size_t
 bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
                const uint8_t *pr, const uint8_t *pa,
                size_t n, uint64_t *sum_io, uint64_t mask)
 {
     uint64_t sum = *sum_io;
-    uint64_t s[8];
+    uint64_t d[16]; /* double buffer: block being tested + block being loaded */
     size_t j = 0;
+    int cur = 0;
     /* _mm256_set_epi64x takes arguments high lane first */
-    __m256i M1 = _mm256_set_epi64x((long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5),
-                                   (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7));
-    __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)BZ_ROTL(mask, 1),
-                                   (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3));
-    __m256i zero = _mm256_setzero_si256();
-
-    while (j + 8 <= n && (sum & mask) != 0) {
-        bz64_block_prefix(T, Trot, pr + j, pa + j, s);
-        uint64_t c = BZ_ROTL(sum, 8);
-        __m256i C = _mm256_set1_epi64x((long long)c);
-        __m256i H1 = _mm256_xor_si256(C, _mm256_loadu_si256((const __m256i *)&s[0]));
-        __m256i H2 = _mm256_xor_si256(C, _mm256_loadu_si256((const __m256i *)&s[4]));
-        __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero);
-        __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero);
-        __m256i any = _mm256_or_si256(z1, z2);
-        if (!_mm256_testz_si256(any, any)) {
-            size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */
-            *sum_io = sum;
-            return j + r;
+    const __m256i M1 = _mm256_set_epi64x((long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5),
+                                         (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7));
+    const __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)BZ_ROTL(mask, 1),
+                                         (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3));
+    const __m256i KL = _mm256_set_epi64x(4, 5, 6, 7); /* lanes 0..3 rotate by 7..4 */
+    const __m256i KH = _mm256_set_epi64x(0, 1, 2, 3); /* lanes 4..7 rotate by 3..0 */
+    const __m256i KLC = _mm256_set_epi64x(60, 59, 58, 57); /* 64 - KL */
+    const __m256i KHC = _mm256_set_epi64x(64, 63, 62, 61); /* 64 - KH */
+    const __m256i zero = _mm256_setzero_si256();
+
+    if (n >= 16 && (sum & mask) != 0) {
+        bz64_block_delta(T, Trot, pr, pa, d);
+        while (j + 16 <= n && (sum & mask) != 0) {
+            bz64_block_delta(T, Trot, pr + j + 8, pa + j + 8, d + (cur ^ 8)); /* one block ahead */
+            __m256i lo = bz64_rotl4(_mm256_loadu_si256((const __m256i *)(d + cur)), KL, KLC);
+            __m256i hi = bz64_rotl4(_mm256_loadu_si256((const __m256i *)(d + cur + 4)), KH, KHC);
+            /* prefix XOR inside each half: rotate the lanes up by 1 resp. 2
+             * (vpermq), zero the lanes rotated in, XOR */
+            lo = _mm256_xor_si256(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x93), zero, 0x03));
+            lo = _mm256_xor_si256(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x4E), zero, 0x0F));
+            hi = _mm256_xor_si256(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x93), zero, 0x03));
+            hi = _mm256_xor_si256(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x4E), zero, 0x0F));
+            hi = _mm256_xor_si256(hi, _mm256_permute4x64_epi64(lo, 0xFF)); /* carry s[3] */
+            uint64_t c = BZ_ROTL(sum, 8);
+            __m256i C = _mm256_set1_epi64x((long long)c);
+            __m256i H1 = _mm256_xor_si256(C, lo);
+            __m256i H2 = _mm256_xor_si256(C, hi);
+            __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero);
+            __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero);
+            __m256i any = _mm256_or_si256(z1, z2);
+            if (!_mm256_testz_si256(any, any)) {
+                size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */
+                *sum_io = sum;
+                return j + r;
+            }
+            sum = c ^ (uint64_t)_mm_cvtsi128_si64(_mm256_castsi256_si128(
+                          _mm256_permute4x64_epi64(hi, 0xFF))); /* s[7] */
+            j += 8;
+            cur ^= 8;
         }
-        sum = c ^ s[7];
-        j += 8;
     }
-    if (j < n)
+    if (j < n) /* up to 15 bytes here (one block more than the scalar kernels) */
         j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask);
     *sum_io = sum;
     return j;
@@ -214,37 +274,56 @@ bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
 
 #define BZ_KIND_512 "avx512"
 
-/* The same 8-lane test as the AVX2 kernel in one 512-bit vector: vptestnmq
- * fuses the AND and the ==0 test per lane into a mask register. Only the
- * test narrows; the scalar prefix work (bz64_block_prefix) is shared. */
+/* The same 8-lane test as the AVX2 kernel, but the aligned domain fits in one
+ * 512-bit vector: vprolvq applies the per-lane rotations (lane k by 7-k) in
+ * one instruction, three valignq+vpxorq steps turn the rotated deltas into
+ * the prefix XORs s[0..7] in one pass instead of two halves plus a carry, and
+ * vptestnmq fuses the AND and the ==0 test into a mask register.
+ *
+ * The table lookups are pipelined one block ahead exactly as in the AVX2
+ * kernel above, for the same store-to-load forwarding reason. */
 __attribute__((target("avx512f"))) static size_t
 bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot,
                   const uint8_t *pr, const uint8_t *pa,
                   size_t n, uint64_t *sum_io, uint64_t mask)
 {
     uint64_t sum = *sum_io;
-    uint64_t s[8];
+    uint64_t d[16]; /* double buffer: block being tested + block being loaded */
     size_t j = 0;
+    int cur = 0;
     /* _mm512_set_epi64 takes arguments high lane first: lane k = ROTL(mask, 7-k) */
-    __m512i M = _mm512_set_epi64((long long)mask, (long long)BZ_ROTL(mask, 1),
-                                 (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3),
-                                 (long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5),
-                                 (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7));
-
-    while (j + 8 <= n && (sum & mask) != 0) {
-        bz64_block_prefix(T, Trot, pr + j, pa + j, s);
-        uint64_t c = BZ_ROTL(sum, 8);
-        __m512i H = _mm512_xor_si512(_mm512_set1_epi64((long long)c),
-                                     _mm512_loadu_si512((const void *)s));
-        if (_mm512_testn_epi64_mask(H, M)) {
-            size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */
-            *sum_io = sum;
-            return j + r;
+    const __m512i M = _mm512_set_epi64((long long)mask, (long long)BZ_ROTL(mask, 1),
+                                       (long long)BZ_ROTL(mask, 2), (long long)BZ_ROTL(mask, 3),
+                                       (long long)BZ_ROTL(mask, 4), (long long)BZ_ROTL(mask, 5),
+                                       (long long)BZ_ROTL(mask, 6), (long long)BZ_ROTL(mask, 7));
+    const __m512i RT = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); /* lane k: ROTL by 7-k */
+    const __m512i Z = _mm512_setzero_si512();
+
+    if (n >= 16 && (sum & mask) != 0) {
+        bz64_block_delta(T, Trot, pr, pa, d);
+        while (j + 16 <= n && (sum & mask) != 0) {
+            bz64_block_delta(T, Trot, pr + j + 8, pa + j + 8, d + (cur ^ 8)); /* one block ahead */
+            /* u = rotated deltas, then the prefix XOR over the 8 lanes
+             * (shift lanes up by 1, 2, 4 and XOR; _mm512_alignr_epi64(u, Z,
+             * 8-k) shifts up by k, zero-filling) */
+            __m512i u = _mm512_rolv_epi64(_mm512_loadu_si512((const void *)(d + cur)), RT);
+            u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 7));
+            u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 6));
+            u = _mm512_xor_si512(u, _mm512_alignr_epi64(u, Z, 4));
+            uint64_t c = BZ_ROTL(sum, 8);
+            __m512i H = _mm512_xor_si512(_mm512_set1_epi64((long long)c), u);
+            if (_mm512_testn_epi64_mask(H, M)) {
+                size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */
+                *sum_io = sum;
+                return j + r;
+            }
+            sum = c ^ (uint64_t)_mm_cvtsi128_si64(_mm512_castsi512_si128(
+                          _mm512_permutexvar_epi64(_mm512_set1_epi64(7), u))); /* s[7] */
+            j += 8;
+            cur ^= 8;
         }
-        sum = c ^ s[7];
-        j += 8;
     }
-    if (j < n)
+    if (j < n) /* up to 15 bytes here (one block more than the other kernels) */
         j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask);
     *sum_io = sum;
     return j;
diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c
index 46dfd141c6..5924ae4304 100644
--- a/src/borg/chunkers/fastcdc_impl.c
+++ b/src/borg/chunkers/fastcdc_impl.c
@@ -173,39 +173,87 @@ static int fc_simd_available(void)
 
 #include <immintrin.h>
 
+/* Load the block's 8 gear values (via one 8-byte data load), without the
+ * shifts and the prefix sum: the vector kernels do those in the vector
+ * domain. Endianness-independent: bytes are extracted by shifting. */
+static inline void fc_block_gear(const uint64_t *gear, const uint8_t *p, uint64_t g[8])
+{
+    uint64_t w;
+    memcpy(&w, p, 8);
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    w = __builtin_bswap64(w);
+#endif
+    g[0] = gear[(uint8_t)w];
+    g[1] = gear[(uint8_t)(w >> 8)];
+    g[2] = gear[(uint8_t)(w >> 16)];
+    g[3] = gear[(uint8_t)(w >> 24)];
+    g[4] = gear[(uint8_t)(w >> 32)];
+    g[5] = gear[(uint8_t)(w >> 40)];
+    g[6] = gear[(uint8_t)(w >> 48)];
+    g[7] = gear[(uint8_t)(w >> 56)];
+}
+
+/* The 8-lane candidate test in two 256-bit vectors, with the aligned domain
+ * built in the vector domain: vpsllvq applies the per-lane shifts, then an
+ * inclusive prefix sum inside each 4-lane half (two vpermq+vpaddq steps) plus
+ * a broadcast of the low half's total into the high half.
+ *
+ * Only the 8 gear table lookups stay scalar, and they are done one block
+ * ahead into a double buffer: a 32-byte vector load placed right after the
+ * eight 8-byte scalar stores that filled it cannot use store-to-load
+ * forwarding and stalls, twice per block. Loading block i+1 while block i is
+ * tested puts a full loop body between the stores and the load, so the stores
+ * have retired by then and the stall disappears. Before this, the stalls made
+ * the AVX2 kernel slower than the blocked scalar one. */
 __attribute__((target("avx2"))) static int64_t
 fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
 {
     uint64_t fp = *fp_io;
-    uint64_t s[8];
+    uint64_t g[16]; /* double buffer: block being tested + block being loaded */
     /* _mm256_set_epi64x takes arguments high lane first */
-    __m256i M1 = _mm256_set_epi64x((long long)(mask << 4), (long long)(mask << 5),
-                                   (long long)(mask << 6), (long long)(mask << 7));
-    __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)(mask << 1),
-                                   (long long)(mask << 2), (long long)(mask << 3));
-    __m256i zero = _mm256_setzero_si256();
+    const __m256i M1 = _mm256_set_epi64x((long long)(mask << 4), (long long)(mask << 5),
+                                         (long long)(mask << 6), (long long)(mask << 7));
+    const __m256i M2 = _mm256_set_epi64x((long long)mask, (long long)(mask << 1),
+                                         (long long)(mask << 2), (long long)(mask << 3));
+    const __m256i SL = _mm256_set_epi64x(4, 5, 6, 7); /* lanes 0..3 <<= 7..4 */
+    const __m256i SH = _mm256_set_epi64x(0, 1, 2, 3); /* lanes 4..7 <<= 3..0 */
+    const __m256i zero = _mm256_setzero_si256();
     size_t i = 0;
-
-    for (; i + 8 <= n; i += 8) {
-        fc_block_prefix(gear, p + i, s);
-        uint64_t c = fp << 8;
-        __m256i C = _mm256_set1_epi64x((long long)c);
-        __m256i H1 = _mm256_add_epi64(C, _mm256_loadu_si256((const __m256i *)&s[0]));
-        __m256i H2 = _mm256_add_epi64(C, _mm256_loadu_si256((const __m256i *)&s[4]));
-        __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero);
-        __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero);
-        __m256i any = _mm256_or_si256(z1, z2);
-        if (!_mm256_testz_si256(any, any)) {
-            int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */
-            if (r >= 0) {
-                *fp_io = fp;
-                return (int64_t)i + r;
+    int cur = 0;
+
+    if (n >= 16) {
+        fc_block_gear(gear, p, g);
+        for (; i + 16 <= n; i += 8, cur ^= 8) {
+            fc_block_gear(gear, p + i + 8, g + (cur ^ 8)); /* one block ahead */
+            __m256i lo = _mm256_sllv_epi64(_mm256_loadu_si256((const __m256i *)(g + cur)), SL);
+            __m256i hi = _mm256_sllv_epi64(_mm256_loadu_si256((const __m256i *)(g + cur + 4)), SH);
+            /* inclusive prefix sum inside each half: rotate the lanes up by 1
+             * resp. 2 (vpermq), zero the lanes rotated in, add */
+            lo = _mm256_add_epi64(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x93), zero, 0x03));
+            lo = _mm256_add_epi64(lo, _mm256_blend_epi32(_mm256_permute4x64_epi64(lo, 0x4E), zero, 0x0F));
+            hi = _mm256_add_epi64(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x93), zero, 0x03));
+            hi = _mm256_add_epi64(hi, _mm256_blend_epi32(_mm256_permute4x64_epi64(hi, 0x4E), zero, 0x0F));
+            hi = _mm256_add_epi64(hi, _mm256_permute4x64_epi64(lo, 0xFF)); /* carry s[3] */
+            uint64_t c = fp << 8;
+            __m256i C = _mm256_set1_epi64x((long long)c);
+            __m256i H1 = _mm256_add_epi64(C, lo);
+            __m256i H2 = _mm256_add_epi64(C, hi);
+            __m256i z1 = _mm256_cmpeq_epi64(_mm256_and_si256(H1, M1), zero);
+            __m256i z2 = _mm256_cmpeq_epi64(_mm256_and_si256(H2, M2), zero);
+            __m256i any = _mm256_or_si256(z1, z2);
+            if (!_mm256_testz_si256(any, any)) {
+                int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */
+                if (r >= 0) {
+                    *fp_io = fp;
+                    return (int64_t)i + r;
+                }
+            } else {
+                fp = c + (uint64_t)_mm_cvtsi128_si64(_mm256_castsi256_si128(
+                             _mm256_permute4x64_epi64(hi, 0xFF))); /* s[7] */
             }
-        } else {
-            fp = c + s[7];
         }
     }
-    if (i < n) {
+    if (i < n) { /* up to 15 bytes here (one block more than the scalar kernels) */
         int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask);
         if (r >= 0) {
             *fp_io = fp;
@@ -220,38 +268,59 @@ fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io,
 
 #define FC_KIND_512 "avx512"
 
-/* The same 8-lane candidate test as the AVX2 kernel, in one 512-bit vector:
- * vptestnmq fuses the AND and the ==0 test per lane into a mask register,
- * so the whole test is broadcast + load + add + testn + branch. Only the
- * test narrows; the scalar prefix work (fc_block_prefix) is shared. */
+/* The same 8-lane candidate test as the AVX2 kernel, but the aligned domain
+ * fits in one 512-bit vector: vpsllvq applies the per-lane shifts (lane j <<
+ * (7-j)), three valignq+vpaddq steps turn the shifted gear values into the
+ * inclusive prefix sums s[0..7] in one pass instead of two halves plus a
+ * carry, and vptestnmq fuses the AND and the ==0 test into a mask register.
+ *
+ * The gear lookups are pipelined one block ahead exactly as in the AVX2
+ * kernel above, for the same store-to-load forwarding reason.
+ *
+ * The serial chain stays short: only fp = c + s[7] (scalar add + shift per
+ * 8 bytes) is carried across blocks, and s[7] is read out of the vector with
+ * vpermq off that chain. */
 __attribute__((target("avx512f"))) static int64_t
 fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
 {
     uint64_t fp = *fp_io;
-    uint64_t s[8];
+    uint64_t g[16]; /* double buffer: block being tested + block being loaded */
     /* _mm512_set_epi64 takes arguments high lane first: lane j = mask << (7-j) */
-    __m512i M = _mm512_set_epi64((long long)mask, (long long)(mask << 1),
-                                 (long long)(mask << 2), (long long)(mask << 3),
-                                 (long long)(mask << 4), (long long)(mask << 5),
-                                 (long long)(mask << 6), (long long)(mask << 7));
+    const __m512i M = _mm512_set_epi64((long long)mask, (long long)(mask << 1),
+                                       (long long)(mask << 2), (long long)(mask << 3),
+                                       (long long)(mask << 4), (long long)(mask << 5),
+                                       (long long)(mask << 6), (long long)(mask << 7));
+    const __m512i SH = _mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7); /* lane j <<= 7-j */
+    const __m512i Z = _mm512_setzero_si512();
     size_t i = 0;
-
-    for (; i + 8 <= n; i += 8) {
-        fc_block_prefix(gear, p + i, s);
-        uint64_t c = fp << 8;
-        __m512i H = _mm512_add_epi64(_mm512_set1_epi64((long long)c),
-                                     _mm512_loadu_si512((const void *)s));
-        if (_mm512_testn_epi64_mask(H, M)) {
-            int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */
-            if (r >= 0) {
-                *fp_io = fp;
-                return (int64_t)i + r;
+    int cur = 0;
+
+    if (n >= 16) {
+        fc_block_gear(gear, p, g);
+        for (; i + 16 <= n; i += 8, cur ^= 8) {
+            fc_block_gear(gear, p + i + 8, g + (cur ^ 8)); /* one block ahead */
+            /* u = shifted gear values, then the inclusive prefix sum over the
+             * 8 lanes (Hillis-Steele: shift lanes up by 1, 2, 4 and add;
+             * _mm512_alignr_epi64(u, Z, 8-k) shifts up by k, zero-filling) */
+            __m512i u = _mm512_sllv_epi64(_mm512_loadu_si512((const void *)(g + cur)), SH);
+            u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 7));
+            u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 6));
+            u = _mm512_add_epi64(u, _mm512_alignr_epi64(u, Z, 4));
+            uint64_t c = fp << 8;
+            __m512i H = _mm512_add_epi64(_mm512_set1_epi64((long long)c), u);
+            if (_mm512_testn_epi64_mask(H, M)) {
+                int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */
+                if (r >= 0) {
+                    *fp_io = fp;
+                    return (int64_t)i + r;
+                }
+            } else {
+                fp = c + (uint64_t)_mm_cvtsi128_si64(_mm512_castsi512_si128(
+                             _mm512_permutexvar_epi64(_mm512_set1_epi64(7), u))); /* s[7] */
             }
-        } else {
-            fp = c + s[7];
         }
     }
-    if (i < n) {
+    if (i < n) { /* up to 15 bytes here (one block more than the other kernels) */
         int64_t r = fc_scan_seq(gear, p + i, n - i, &fp, mask);
         if (r >= 0) {
             *fp_io = fp;

From aa21d6f037f669de4515eeb4b2ceeed2e1f621f5 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 08:44:00 +0200
Subject: [PATCH 029/144] goldilocks-aes: keep the field reductions out of the
 branch predictor

gl_add and gl_mul reduce with plain if() statements. The conditions are
data-dependent and essentially random, and GCC turns them into real
conditional jumps - 55 of them in the VAES scan kernel, missing about 27%
of the time. The mispredicts, not the arithmetic, were what made this
chunker slow: a single gl_mul cost 15.1 cycles, against 1.8 for the raw
64x64 multiply it is built on and 3.1 for the same code with the final
canonicalisation removed.

Writing the reductions as mask arithmetic or as a ternary does not help:
GCC recognises both and converts them straight back to branches (measured
identical miss counts). The __builtin_*_overflow forms do, because the
carry/borrow stays in the flags and the compiler settles on sbb/adc plus a
mask. gl_mul drops to 4.4 cycles and the kernel keeps 5 conditional jumps
instead of 55.

Everything stays canonical, which matters because the state is fed to AES
verbatim - a different representation of the same field element would
change cut decisions. Cut points are bit-identical: the chunker tests and
the fuzz tests pass, and the chunk-boundary digest is unchanged across the
vaes, aes-ni and evp paths.

Measured on an AMD Ryzen 5 8500GE, borg benchmark cpu, 1 GB, both states
built and measured back to back, pinned to one core, best of two passes:

    goldilocks-aes   7.121s -> 2.164s  (3.29x)

Isolated kernel, same box: 94.0 -> 336.9 MB/s.

This is the rolling hash only, so it applies to all three scan paths, not
just the VAES one. rabin-aes and toeplitz-aes are unaffected: their rolling
hashes are GF(2)[x] and have no data-dependent reduction.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/goldilocks_aes_impl.c | 34 +++++++++++++------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/borg/chunkers/goldilocks_aes_impl.c b/src/borg/chunkers/goldilocks_aes_impl.c
index 5b045e89fc..423f76669f 100644
--- a/src/borg/chunkers/goldilocks_aes_impl.c
+++ b/src/borg/chunkers/goldilocks_aes_impl.c
@@ -35,14 +35,19 @@ struct GL_CTX {
  * AES verbatim, so a non-canonical representation of the same field element
  * would change cut decisions. */
 
+/* The reductions are data-dependent and unpredictable (~27% branch-miss rate
+ * measured), so they must not become branches. Writing them as if() or as a
+ * ternary lets GCC emit real conditional jumps - the mispredicts alone cost
+ * more than the arithmetic (gl_mul: 15.1 vs 4.4 cycles per call). The
+ * __builtin_*_overflow forms keep the carry/borrow in the flags, so the
+ * compiler settles on sbb/adc + mask instead. Results are unchanged: the
+ * value is still fully canonical, which matters because it is fed to AES. */
 static inline uint64_t gl_add(uint64_t a, uint64_t b)
 {
-    uint64_t s = a + b;
-    if (s < a)          /* carry out: 2^64 mod p = GL_EPS; cannot re-carry */
-        s += GL_EPS;
-    if (s >= GL_P)
-        s -= GL_P;
-    return s;
+    uint64_t s, u;
+    unsigned char carry = __builtin_add_overflow(a, b, &s);
+    s += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* 2^64 mod p; cannot re-carry */
+    return __builtin_sub_overflow(s, GL_P, &u) ? s : u;
 }
 
 /* a * b mod p via the standard 2^64 = 2^32 - 1 folding (risc0/plonky2 style):
@@ -53,16 +58,13 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b)
     __uint128_t t = (__uint128_t)a * b;
     uint64_t lo = (uint64_t)t, hi = (uint64_t)(t >> 64);
     uint64_t hi_hi = hi >> 32, hi_lo = hi & GL_EPS;
-    uint64_t t0 = lo - hi_hi;
-    if (lo < hi_hi)     /* borrow: -2^64 = -eps (mod p) */
-        t0 -= GL_EPS;
-    uint64_t t1 = hi_lo * GL_EPS; /* < 2^64 - 2^33 + 2, no overflow */
-    uint64_t r = t0 + t1;
-    if (r < t1)         /* carry out; cannot re-carry (t1 <= 2^64 - 2^33 + 1) */
-        r += GL_EPS;
-    if (r >= GL_P)
-        r -= GL_P;
-    return r;
+    uint64_t t0, r, u;
+    unsigned char borrow = __builtin_sub_overflow(lo, hi_hi, &t0);
+    t0 -= ((uint64_t)0 - (uint64_t)borrow) & GL_EPS; /* -2^64 = -eps (mod p) */
+    /* t1 = hi_lo * eps < 2^64 - 2^33 + 2, no overflow */
+    unsigned char carry = __builtin_add_overflow(t0, hi_lo * GL_EPS, &r);
+    r += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* cannot re-carry */
+    return __builtin_sub_overflow(r, GL_P, &u) ? r : u;
 }
 
 /* Advance the state by one byte. The state is the window's polynomial hash

From 202088852156fa759a9762d7aa01ce05082faadf Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 09:39:07 +0200
Subject: [PATCH 030/144] chunkers: store the AES chunkers' digests pre-spread

An AES input block wants the digest in its low qword and zero in its high
one. The VAES kernel stored the 32 digests of a group packed and spread
them at use with vpexpandq, eight of those per group.

Storing them pre-spread costs nothing extra: the odd slots hold the zeros,
they are written once when the scan starts and never touched again, and the
chain writes the same 32 values it wrote before - just at 2j instead of j.
Each vector's AES input then becomes a plain 512-bit load.

Measured on an AMD Ryzen 5 8500GE, isolated scan kernel, pinned, best of 5:

    toeplitz-aes     652.9 -> 712.8 MB/s   (+9.2%)
    rabin-aes        650.0 -> 690.8 MB/s   (+6.3%)
    goldilocks-aes   337.3 -> 347.2 MB/s   (+2.9%)

borg benchmark cpu, 1 GB, both states built and measured back to back,
pinned, best of three passes:

    toeplitz-aes     1.250s -> 1.078s  (1.16x)
    rabin-aes        1.176s -> 1.107s  (1.06x)
    goldilocks-aes   2.160s -> 2.192s  (unchanged; the passes overlap)

goldilocks-aes is dominated by its field arithmetic, so it barely notices
either way. Cut points are bit-identical: the chunker and fuzz tests pass
and the chunk-boundary digests are unchanged on all three chunkers across
the vaes, aes-ni and evp paths.

This is the only one of several attempts that paid off. For the record, on
this CPU the following were measured and rejected: widening the rolling
hash to four lanes at stride 4 (toeplitz 653 -> 402, rabin 649 -> 458 MB/s,
because stride-4 doubles the table footprint against a 32 KiB L1d while the
lookups per position stay the same); hoisting the chain-independent table
lookups into their own pass (653 -> 579 and 650 -> 495, the delta buffer's
traffic costs more than the scheduling gain, so the hardware was already
hoisting them); and rolling the digest buffer one group ahead (+2.5% and
+1%, not worth the complexity). A microbenchmark of the rolling step alone
puts the ceiling for extra lanes at 16% for rabin and nothing for toeplitz,
which is why the four-lane attempt could not have won.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/phte_scan.h | 52 +++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 21 deletions(-)

diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h
index 776b2360da..2492b602a6 100644
--- a/src/borg/chunkers/phte_scan.h
+++ b/src/borg/chunkers/phte_scan.h
@@ -344,16 +344,25 @@ PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t
  * vectors of 4 AES blocks each - 4x fewer AES instructions than the 128-bit
  * path, the round keys stay register-resident (32 zmm registers instead of
  * 16 xmm, so no per-round key reloads), and the 8 independent chains hide
- * the vaesenc latency (2 chains would be latency-bound). vpexpandq places a
- * vector's 4 digests into its block-low qwords in one instruction, and a
- * masked vptestnmq tests the 4 ciphertext-low qwords against the mask
- * without extracting. Every digest is exact and every position is tested,
- * so cut points stay bit-identical to the other paths. */
+ * the vaesenc latency (2 chains would be latency-bound), and a masked
+ * vptestnmq tests the 4 ciphertext-low qwords against the mask without
+ * extracting. Every digest is exact and every position is tested, so cut
+ * points stay bit-identical to the other paths.
+ *
+ * The digests are stored pre-spread: an AES input block wants the digest in
+ * the low qword and zero in the high one, so dgs[] keeps a digest in every
+ * even slot and a zero in every odd one. The zeros are written once at entry
+ * and never touched again, which makes each vector's input a plain aligned
+ * 512-bit load. Packing the digests and spreading them with vpexpandq (eight
+ * of those per group) instead costs the same stores and measurably more
+ * time - about 9% of the whole toeplitz-aes scan, 6% of rabin-aes. */
 __attribute__((target("aes,sse2,vaes,avx512f"))) static int64_t
 PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask)
 {
     __m512i k[11];
     __m512i M = _mm512_set1_epi64((long long)mask);
+    /* digest at dgs[2j], zero at dgs[2j+1] (= AES input block bytes 8..15) */
+    uint64_t dgs[64] __attribute__((aligned(64)));
     uint64_t d = *digest, da, db;
     const uint8_t *qo = p - 64;
     size_t i = 0;
@@ -362,6 +371,8 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6
     /* broadcast each 128-bit round key to all four block lanes */
     for (int j = 0; j < 11; j++)
         k[j] = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c->base.rk[j]));
+    for (int j = 1; j < 64; j += 2)
+        dgs[j] = 0;
 
     if (n >= 34) {
         db = PH_ROLL(c, d, qo[0], p[0]);               /* d_0 */
@@ -370,32 +381,31 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6
     }
 
     while (lanes_live && i + 34 <= n) {
-        uint64_t dg[32];
         __m512i b0, b1, b2, b3, b4, b5, b6, b7;
         __mmask8 h0, h1, h2, h3, h4, h5, h6, h7;
 
-        dg[0] = db;
-        dg[1] = da;
+        dgs[0] = db;
+        dgs[2] = da;
         for (int j = 2; j < 32; j += 2) {
             db = PH_ROLL2(c, db, qo[i + j - 1], qo[i + j], p[i + j - 1], p[i + j]);
-            dg[j] = db;
+            dgs[2 * j] = db;
             da = PH_ROLL2(c, da, qo[i + j], qo[i + j + 1], p[i + j], p[i + j + 1]);
-            dg[j + 1] = da;
+            dgs[2 * j + 2] = da;
         }
         /* prepare the next group's invariant (digests at i+32, i+33) */
         db = PH_ROLL2(c, db, qo[i + 31], qo[i + 32], p[i + 31], p[i + 32]);
         da = PH_ROLL2(c, da, qo[i + 32], qo[i + 33], p[i + 32], p[i + 33]);
 
-        /* qword lanes 0,2,4,6 take 4 digests, lanes 1,3,5,7 stay zero
-         * (= AES input block bytes 8..15, as in the other paths) */
-        b0 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg), k[0]);
-        b1 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 4), k[0]);
-        b2 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 8), k[0]);
-        b3 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 12), k[0]);
-        b4 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 16), k[0]);
-        b5 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 20), k[0]);
-        b6 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 24), k[0]);
-        b7 = _mm512_xor_si512(_mm512_maskz_expandloadu_epi64(0x55, dg + 28), k[0]);
+        /* each vector's 4 blocks are already laid out in dgs (digest in the
+         * low qword of every block, zero in the high one) */
+        b0 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 0)), k[0]);
+        b1 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 8)), k[0]);
+        b2 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 16)), k[0]);
+        b3 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 24)), k[0]);
+        b4 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 32)), k[0]);
+        b5 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 40)), k[0]);
+        b6 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 48)), k[0]);
+        b7 = _mm512_xor_si512(_mm512_loadu_si512((const void *)(dgs + 56)), k[0]);
         for (int r = 1; r < 10; r++) {
             b0 = _mm512_aesenc_epi128(b0, k[r]);
             b1 = _mm512_aesenc_epi128(b1, k[r]);
@@ -420,7 +430,7 @@ PH_FN(scan_hw512)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *digest, uint6
             const uint8_t hs[8] = {h0, h1, h2, h3, h4, h5, h6, h7};
             for (int j = 0; j < 32; j++) {
                 if ((hs[j >> 2] >> ((j & 3) * 2)) & 1) {
-                    *digest = dg[j];
+                    *digest = dgs[2 * j];
                     return (int64_t)(i + j);
                 }
             }

From a9e5d032a41ee8142d16dee473f60bf645069301 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 11:30:10 +0200
Subject: [PATCH 031/144] fastcdc: test against a limit instead of masking, on
 aarch64

Per lane the NEON kernel tested "(c + s_j) & M_j == 0". Because
(x & M) == 0 means x's set bits are a subset of ~M's, which implies
x <= ~M, the and plus compare-against-zero can be one unsigned compare
against a limit computed once per scan. It never loses a cut for any
mask, and for the contiguous high-bit masks fastcdc uses it is exactly
equivalent - still the same superset test the block's sequential
recheck resolves.

Apple M3 Pro, 128 MiB of random data, chunkify end to end:
3088 -> 3431 MB/s (1.11x). Cut points unchanged (verified against the
sequential kernel).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/fastcdc_impl.c | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c
index 5924ae4304..fe2c49a8d3 100644
--- a/src/borg/chunkers/fastcdc_impl.c
+++ b/src/borg/chunkers/fastcdc_impl.c
@@ -121,24 +121,35 @@ static int64_t fc_scan_blocked(const uint64_t *gear, const uint8_t *p, size_t n,
 
 #include <arm_neon.h>
 
+/* Per lane, "(x & M) == 0" is replaced by "x <= ~M", turning the and plus
+ * compare-against-zero into one unsigned compare against a limit computed
+ * once per scan.
+ *
+ * (x & M) == 0 means x's set bits are a subset of ~M's, which implies
+ * x <= ~M for any M - so this never loses a cut, whatever the mask looks
+ * like. For the contiguous high-bit masks the chunker actually uses (mask
+ * has its one-bits at the top and shifting left only drops bits off the top,
+ * so ~M is 2^(64-k)-1) the two are exactly equivalent, and this stays the
+ * same superset test as the masked form: the block's exact sequential
+ * recheck resolves candidates either way. */
 static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
 {
     uint64_t fp = *fp_io;
     uint64_t s[8];
-    uint64x2_t M12 = {mask << 7, mask << 6};
-    uint64x2_t M34 = {mask << 5, mask << 4};
-    uint64x2_t M56 = {mask << 3, mask << 2};
-    uint64x2_t M78 = {mask << 1, mask};
+    uint64x2_t L12 = {~(mask << 7), ~(mask << 6)};
+    uint64x2_t L34 = {~(mask << 5), ~(mask << 4)};
+    uint64x2_t L56 = {~(mask << 3), ~(mask << 2)};
+    uint64x2_t L78 = {~(mask << 1), ~mask};
     size_t i = 0;
 
     for (; i + 8 <= n; i += 8) {
         fc_block_prefix(gear, p + i, s);
         uint64_t c = fp << 8;
         uint64x2_t C = vdupq_n_u64(c);
-        uint64x2_t z12 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[0], s[1]}), M12));
-        uint64x2_t z34 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[2], s[3]}), M34));
-        uint64x2_t z56 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[4], s[5]}), M56));
-        uint64x2_t z78 = vceqzq_u64(vandq_u64(vaddq_u64(C, (uint64x2_t){s[6], s[7]}), M78));
+        uint64x2_t z12 = vcgeq_u64(L12, vaddq_u64(C, (uint64x2_t){s[0], s[1]}));
+        uint64x2_t z34 = vcgeq_u64(L34, vaddq_u64(C, (uint64x2_t){s[2], s[3]}));
+        uint64x2_t z56 = vcgeq_u64(L56, vaddq_u64(C, (uint64x2_t){s[4], s[5]}));
+        uint64x2_t z78 = vcgeq_u64(L78, vaddq_u64(C, (uint64x2_t){s[6], s[7]}));
         uint64x2_t any = vorrq_u64(vorrq_u64(z12, z34), vorrq_u64(z56, z78));
         if (vmaxvq_u32(vreinterpretq_u32_u64(any))) {
             int64_t r = fc_scan_seq(gear, p + i, 8, &fp, mask); /* exact recheck */

From 193a26f4e9f3ade4708258f5c9768c194953c410 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 11:30:45 +0200
Subject: [PATCH 032/144] buzhash64: drop the NEON kernel, blocked scalar is
 faster there

The NEON kernel lost to the blocked scalar one on Apple M3 Pro: 2323 vs
2519 MB/s end to end, and in an isolated kernel benchmark 2360 vs 2590
MB/s with the same ~9% gap at every mask size from 17 to 23 bits. So
aarch64 now falls through to the blocked scalar kernel.

Nothing in this test wants a vector register. clang keeps the block's
eight prefix XORs in general registers, and xor/and/compare per lane
retires at more than one lane per cycle on the wide scalar ALUs; the
vector form first has to move those eight values to the SIMD side and
then reduce the 8-lane result back to a condition flag (umaxv) before
the loop branch can resolve. fastcdc keeps its NEON kernel because its
per-lane work is larger (add plus per-lane shifted masks) - enough to
pay for the trip, and there it wins by 2x.

Cut points unchanged - this only changes which bit-identical kernel
runs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/buzhash64_impl.c | 77 +++++++++---------------------
 1 file changed, 23 insertions(+), 54 deletions(-)

diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index e54f45374b..b5c05c57c6 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -21,13 +21,9 @@
  * Trot[b] = ROTL(T[b], window_size % 64) is precomputed by the caller, which
  * also removes one rotate per byte from the sequential path.
  *
- * Kernel dispatch mirrors fastcdc_impl.c: NEON on aarch64 (baseline there),
- * AVX-512 or AVX2 on x86-64 (runtime-detected), blocked scalar elsewhere,
- * sequential when forced. Measured on Apple M-series (64 MiB, window 4095, 21-bit
- * mask): sequential 1150, blocked scalar 2340, NEON 2260 MB/s - NEON and
- * blocked are a tie here (throughput and energy, within noise), because the
- * XOR/AND/compare test also runs well on the scalar ALUs; the symmetric
- * dispatch is kept for consistency across the SIMD chunker kernels.
+ * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blocked
+ * scalar everywhere else including aarch64 (which has no vector kernel, see
+ * the note where it would go), sequential when forced.
  * All kernels return bit-identical results. */
 
 #include <stdlib.h>
@@ -118,56 +114,29 @@ static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
-/* --- NEON (aarch64 baseline, always available) -------------------------- */
-
-#if defined(__aarch64__)
-#define BZ_KIND "neon"
-
-#include <arm_neon.h>
-
-static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
-                             const uint8_t *pr, const uint8_t *pa,
-                             size_t n, uint64_t *sum_io, uint64_t mask)
-{
-    uint64_t sum = *sum_io;
-    uint64_t s[8];
-    size_t j = 0;
-    uint64x2_t M12 = {BZ_ROTL(mask, 7), BZ_ROTL(mask, 6)};
-    uint64x2_t M34 = {BZ_ROTL(mask, 5), BZ_ROTL(mask, 4)};
-    uint64x2_t M56 = {BZ_ROTL(mask, 3), BZ_ROTL(mask, 2)};
-    uint64x2_t M78 = {BZ_ROTL(mask, 1), mask};
-
-    while (j + 8 <= n && (sum & mask) != 0) {
-        bz64_block_prefix(T, Trot, pr + j, pa + j, s);
-        uint64_t c = BZ_ROTL(sum, 8);
-        uint64x2_t Cv = vdupq_n_u64(c);
-        uint64x2_t z12 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[0], s[1]}), M12));
-        uint64x2_t z34 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[2], s[3]}), M34));
-        uint64x2_t z56 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[4], s[5]}), M56));
-        uint64x2_t z78 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[6], s[7]}), M78));
-        uint64x2_t any = vorrq_u64(vorrq_u64(z12, z34), vorrq_u64(z56, z78));
-        if (vmaxvq_u32(vreinterpretq_u32_u64(any))) {
-            size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */
-            *sum_io = sum;
-            return j + r;
-        }
-        sum = c ^ s[7];
-        j += 8;
-    }
-    if (j < n)
-        j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask);
-    *sum_io = sum;
-    return j;
-}
-
-static int bz64_simd_available(void)
-{
-    return 1;
-}
+/* --- aarch64: no vector kernel ------------------------------------------
+ *
+ * There was a NEON kernel here doing the 8-lane test in vector registers.
+ * It is gone because it lost to the blocked scalar kernel: measured on an
+ * Apple M3 Pro (48 MiB, window 4095), blocked 2590 MB/s vs NEON 2360 MB/s,
+ * the same ~9% gap at every mask size from 17 to 23 bits.
+ *
+ * The reason is that nothing here needs a vector register in the first
+ * place. clang keeps the block's eight prefix XORs in general registers and
+ * the scalar test is xor/and/compare per lane, which the wide scalar ALUs
+ * retire at more than one lane per cycle; the vector form has to move those
+ * eight values across to the SIMD side and then reduce the 8-lane result
+ * back to one condition flag (umaxv, ~4 cycles) before the loop branch can
+ * resolve. The move-and-reduce round trip costs more than the parallel test
+ * saves. fastcdc keeps its NEON kernel because its test is add/and/compare
+ * with per-lane shifted masks - enough extra per-lane work to pay for the
+ * trip - and there it wins by 2x.
+ *
+ * So aarch64 falls through to the blocked scalar kernel below. */
 
 /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */
 
-#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
 #define BZ_KIND "avx2"
 
 #include <immintrin.h>

From 063edc015964481dc97aa6dc7ca53c2d52e386a6 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 11:31:00 +0200
Subject: [PATCH 033/144] chunkers: keep the AES chunkers' cut test in the
 vector domain on aarch64

The aarch64 scan moved all 8 ciphertexts to general registers (one fmov
each) and tested them with 8 scalar and/compare pairs, even though the
common path needs nothing but the single "did any lane hit" bit. uzp1
packs the low halves - the cut-decision uint64 - of two blocks into one
vector, so the test is 4 and/compare-zero pairs, an or-reduce and one
umaxv; the 8 values are only materialised on an actual hit.

Apple M3 Pro, 128 MiB of random data, chunkify end to end:
  toeplitz-aes    866 -> 921 MB/s (1.06x)
  rabin-aes       854 -> 912 MB/s (1.07x)
  goldilocks-aes  461 -> 522 MB/s (1.13x)

Cut points unchanged (verified against the EVP path).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/phte_scan.h | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/src/borg/chunkers/phte_scan.h b/src/borg/chunkers/phte_scan.h
index 2492b602a6..1362658099 100644
--- a/src/borg/chunkers/phte_scan.h
+++ b/src/borg/chunkers/phte_scan.h
@@ -181,18 +181,28 @@ static int64_t PH_FN(scan_hw)(PH_CTX *c, const uint8_t *p, size_t n, uint64_t *d
         b6 = veorq_u8(vaeseq_u8(b6, k[9]), k[10]);
         b7 = veorq_u8(vaeseq_u8(b7, k[9]), k[10]);
 
+        /* Test all 8 ciphertexts without leaving the vector domain: uzp1
+         * packs the low halves (the cut-decision uint64) of two blocks into
+         * one vector, then and/cmeq-zero/or-reduce. Moving the 8 values to
+         * general registers instead - one fmov each, plus 8 scalar and/cmp
+         * pairs - costs noticeably more on the common no-hit path, where
+         * nothing but the single "any lane hit" bit is ever needed. */
         {
-            uint64_t c0 = vgetq_lane_u64(vreinterpretq_u64_u8(b0), 0);
-            uint64_t c1 = vgetq_lane_u64(vreinterpretq_u64_u8(b1), 0);
-            uint64_t c2 = vgetq_lane_u64(vreinterpretq_u64_u8(b2), 0);
-            uint64_t c3 = vgetq_lane_u64(vreinterpretq_u64_u8(b3), 0);
-            uint64_t c4 = vgetq_lane_u64(vreinterpretq_u64_u8(b4), 0);
-            uint64_t c5 = vgetq_lane_u64(vreinterpretq_u64_u8(b5), 0);
-            uint64_t c6 = vgetq_lane_u64(vreinterpretq_u64_u8(b6), 0);
-            uint64_t c7 = vgetq_lane_u64(vreinterpretq_u64_u8(b7), 0);
-            if ((((c0 & mask) == 0) | ((c1 & mask) == 0) | ((c2 & mask) == 0) | ((c3 & mask) == 0) |
-                 ((c4 & mask) == 0) | ((c5 & mask) == 0) | ((c6 & mask) == 0) | ((c7 & mask) == 0))) {
-                uint64_t cs[8] = {c0, c1, c2, c3, c4, c5, c6, c7};
+            const uint64x2_t maskv = vdupq_n_u64(mask);
+            uint64x2_t h01 = vuzp1q_u64(vreinterpretq_u64_u8(b0), vreinterpretq_u64_u8(b1));
+            uint64x2_t h23 = vuzp1q_u64(vreinterpretq_u64_u8(b2), vreinterpretq_u64_u8(b3));
+            uint64x2_t h45 = vuzp1q_u64(vreinterpretq_u64_u8(b4), vreinterpretq_u64_u8(b5));
+            uint64x2_t h67 = vuzp1q_u64(vreinterpretq_u64_u8(b6), vreinterpretq_u64_u8(b7));
+            uint64x2_t any = vorrq_u64(vorrq_u64(vceqzq_u64(vandq_u64(h01, maskv)),
+                                                vceqzq_u64(vandq_u64(h23, maskv))),
+                                       vorrq_u64(vceqzq_u64(vandq_u64(h45, maskv)),
+                                                vceqzq_u64(vandq_u64(h67, maskv))));
+            if (vmaxvq_u32(vreinterpretq_u32_u64(any))) {
+                uint64_t cs[8];
+                vst1q_u64(cs + 0, h01);
+                vst1q_u64(cs + 2, h23);
+                vst1q_u64(cs + 4, h45);
+                vst1q_u64(cs + 6, h67);
                 for (int j = 0; j < 8; j++) {
                     if ((cs[j] & mask) == 0) {
                         *digest = dg[j];

From e310b896f3473dd087ab63b9045489b437d7d418 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 11:59:20 +0200
Subject: [PATCH 034/144] goldilocks-aes: drop the multiply's redundant
 canonicalization

gl_mul ended with a conditional subtraction of p to return a canonical
field element, but both callers feed the result straight into a gl_add
that canonicalizes anyway. gl_mul_lazy skips it and returns a merely
reduced representative (< 2^64 rather than < p).

gl_add still returns canonical with a non-canonical first operand: for
a < 2^64 and b < p a carry leaves a + b - 2^64 <= p - 2, so adding eps
cannot carry a second time, and the single conditional subtraction of p
then lands below p (r - p < 2^32). The digest handed to AES therefore
stays canonical, which it must be - it is encrypted verbatim, so a
different representative of the same element would move cut points.

This kernel is bound by the multiply latency chain (~17 cycles per
stride-2 roll, two lanes, which alone accounts for its throughput), so
removing the trailing compare and select from that chain is worth more
than the op count suggests.

Apple M3 Pro, 128 MiB of random data, chunkify end to end:
517 -> 561 MB/s (1.08x). Cut points unchanged (identical chunk-boundary
digests, and identical over 48 MiB at 17/19/21/23 mask bits against the
canonical version).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/goldilocks_aes_impl.c | 36 +++++++++++++++++++++----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/borg/chunkers/goldilocks_aes_impl.c b/src/borg/chunkers/goldilocks_aes_impl.c
index 423f76669f..f808a59ebe 100644
--- a/src/borg/chunkers/goldilocks_aes_impl.c
+++ b/src/borg/chunkers/goldilocks_aes_impl.c
@@ -31,9 +31,11 @@ struct GL_CTX {
 
 /* --- Goldilocks field arithmetic ---------------------------------------
  *
- * All values are kept canonical (< p) at every step: the state is fed to
- * AES verbatim, so a non-canonical representation of the same field element
- * would change cut decisions. */
+ * Every digest the scan produces is canonical (< p): it is fed to AES
+ * verbatim, so a non-canonical representation of the same field element
+ * would change cut decisions. The rolls reach that via gl_add, which always
+ * canonicalizes; only the multiply inside them is allowed to hand on a
+ * merely reduced (< 2^64) representative, see gl_mul_lazy. */
 
 /* The reductions are data-dependent and unpredictable (~27% branch-miss rate
  * measured), so they must not become branches. Writing them as if() or as a
@@ -67,6 +69,30 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b)
     return __builtin_sub_overflow(r, GL_P, &u) ? r : u;
 }
 
+/* gl_mul without the final canonicalization: the result is congruent to a*b
+ * mod p but only bounded by 2^64, not by p.
+ *
+ * That is enough for the rolls, because both feed it straight into a gl_add
+ * whose other operand is canonical, and that gl_add canonicalizes anyway.
+ * The bound still holds with a non-canonical first operand: for a < 2^64 and
+ * b < p, a carry leaves a + b - 2^64 <= p - 2, so adding eps cannot carry a
+ * second time, and the one conditional subtraction of p then lands below p
+ * (r - p < 2^32). So the digest handed to AES stays canonical - which it must
+ * be, since it is fed to AES verbatim - while the multiply chain that limits
+ * this kernel loses its trailing compare and select. */
+static inline uint64_t gl_mul_lazy(uint64_t a, uint64_t b)
+{
+    __uint128_t t = (__uint128_t)a * b;
+    uint64_t lo = (uint64_t)t, hi = (uint64_t)(t >> 64);
+    uint64_t hi_hi = hi >> 32, hi_lo = hi & GL_EPS;
+    uint64_t t0, r;
+    unsigned char borrow = __builtin_sub_overflow(lo, hi_hi, &t0);
+    t0 -= ((uint64_t)0 - (uint64_t)borrow) & GL_EPS; /* -2^64 = -eps (mod p) */
+    unsigned char carry = __builtin_add_overflow(t0, hi_lo * GL_EPS, &r);
+    r += ((uint64_t)0 - (uint64_t)carry) & GL_EPS; /* cannot re-carry */
+    return r;
+}
+
 /* Advance the state by one byte. The state is the window's polynomial hash
  * s = sum b_j * K^(63-j) (oldest byte at K^63, newest at K^0), so
  *   s' = s*K - byte_out*K^64 + byte_in.
@@ -75,7 +101,7 @@ static inline uint64_t gl_mul(uint64_t a, uint64_t b)
 static inline uint64_t gl_roll(const GL_CTX *c, uint64_t s, uint8_t byte_out, uint8_t byte_in)
 {
     uint64_t delta = gl_add(c->nout64_tbl[byte_out], byte_in);
-    return gl_add(gl_mul(s, c->k1), delta);
+    return gl_add(gl_mul_lazy(s, c->k1), delta);
 }
 
 /* Advance the state by TWO bytes in one step (exact composition of two
@@ -89,7 +115,7 @@ static inline uint64_t gl_roll2(const GL_CTX *c, uint64_t s,
 {
     uint64_t delta = gl_add(gl_add(c->nout65_tbl[o0], c->nout64_tbl[o1]),
                             gl_add(c->in1_tbl[i0], i1));
-    return gl_add(gl_mul(s, c->k2), delta);
+    return gl_add(gl_mul_lazy(s, c->k2), delta);
 }
 
 uint64_t gl_digest64(const GL_CTX *c, const uint8_t *q)

From 18744d69a1bc6711fc09b27e3ad31ae8e591f381 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 12:17:52 +0200
Subject: [PATCH 035/144] chunkers: rename the "blocked" scan kernel to
 "blockwise"

"blocked" is a past participle and reads as "obstructed" or "disabled"
rather than "processed in blocks". The neighbouring API makes that
misreading easy to land on: after setting BORG_FASTCDC_NO_AVX2=1, which
does block AVX2, the chunker reports kernel == "blocked", which parses
as "the kernel was blocked" - a wrong reading that happens to be almost
right, so nothing corrects it. The exported name also dropped the word
that carried the meaning: the code says "blocked scalar" in prose but
exported only "blocked".

"blockwise" says block by block and cannot mean obstructed. The name has
not been released (it was added after 2.0.0b22), so nothing depends on
it.

Also drops 'neon' from buzhash64's documented kernel values and from the
changelog: buzhash64 has no vector kernel on aarch64 any more, so that
value can no longer be returned.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                              |  5 ++--
 src/borg/chunkers/buzhash64.pyx               |  2 +-
 src/borg/chunkers/buzhash64_impl.c            | 24 +++++++++----------
 src/borg/chunkers/buzhash64_impl.h            |  5 ++--
 src/borg/chunkers/fastcdc.pyx                 |  6 ++---
 src/borg/chunkers/fastcdc_impl.c              | 20 ++++++++--------
 src/borg/chunkers/fastcdc_impl.h              |  2 +-
 src/borg/testsuite/chunkers/buzhash64_test.py |  4 ++--
 src/borg/testsuite/chunkers/fastcdc_test.py   |  4 ++--
 9 files changed, 37 insertions(+), 35 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 7c6a90d785..28872ef66b 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -188,9 +188,10 @@ New features:
   - fastcdc is the new and faster default chunker, #9957
   - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043:
 
-    - NEON on aarch64 (e.g. Apple Silicon)
     - AVX-512 / AVX2 on x86-64 (Intel / AMD)
-    - blocked scalar elsewhere
+    - fastcdc also NEON on aarch64 (e.g. Apple Silicon); buzhash64 is faster
+      blockwise there, so it has no vector kernel on aarch64
+    - blockwise scalar elsewhere
   - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers
     (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987
   - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64
diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx
index 4e7b75a40a..03874b92c9 100644
--- a/src/borg/chunkers/buzhash64.pyx
+++ b/src/borg/chunkers/buzhash64.pyx
@@ -155,7 +155,7 @@ cdef class ChunkerBuzHash64(ChunkerBase):
 
     @property
     def kernel(self):
-        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'."""
+        """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'."""
         return (<bytes>bz64_kernel_name(self.force_scalar)).decode("ascii")
 
     cdef object process(self):
diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index b5c05c57c6..27fdbcd0ad 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -21,7 +21,7 @@
  * Trot[b] = ROTL(T[b], window_size % 64) is precomputed by the caller, which
  * also removes one rotate per byte from the sequential path.
  *
- * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blocked
+ * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blockwise
  * scalar everywhere else including aarch64 (which has no vector kernel, see
  * the note where it would go), sequential when forced.
  * All kernels return bit-identical results. */
@@ -82,9 +82,9 @@ static inline void bz64_block_prefix(const uint64_t *T, const uint64_t *Trot,
     s[7] = s8;
 }
 
-/* --- blocked scalar (the default kernel) --------------------------------- */
+/* --- blockwise scalar (the default kernel) --------------------------------- */
 
-static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot,
+static size_t bz64_scan_blockwise(const uint64_t *T, const uint64_t *Trot,
                                 const uint8_t *pr, const uint8_t *pa,
                                 size_t n, uint64_t *sum_io, uint64_t mask)
 {
@@ -117,8 +117,8 @@ static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot,
 /* --- aarch64: no vector kernel ------------------------------------------
  *
  * There was a NEON kernel here doing the 8-lane test in vector registers.
- * It is gone because it lost to the blocked scalar kernel: measured on an
- * Apple M3 Pro (48 MiB, window 4095), blocked 2590 MB/s vs NEON 2360 MB/s,
+ * It is gone because it lost to the blockwise scalar kernel: measured on an
+ * Apple M3 Pro (48 MiB, window 4095), blockwise 2590 MB/s vs NEON 2360 MB/s,
  * the same ~9% gap at every mask size from 17 to 23 bits.
  *
  * The reason is that nothing here needs a vector register in the first
@@ -132,7 +132,7 @@ static size_t bz64_scan_blocked(const uint64_t *T, const uint64_t *Trot,
  * with per-lane shifted masks - enough extra per-lane work to pay for the
  * trip - and there it wins by 2x.
  *
- * So aarch64 falls through to the blocked scalar kernel below. */
+ * So aarch64 falls through to the blockwise scalar kernel below. */
 
 /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */
 
@@ -307,7 +307,7 @@ static int bz64_env_set(const char *name)
 static int bz64_simd_available(void)
 {
     /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2, BORG_BUZHASH64_NO_AVX2
-     * at the blocked scalar kernel (for benchmarking the kernels against
+     * at the blockwise scalar kernel (for benchmarking the kernels against
      * each other); like the detection itself they are read once per process,
      * so they must be set before the first chunker use. */
     if (bz64_env_set("BORG_BUZHASH64_NO_AVX2"))
@@ -318,13 +318,13 @@ static int bz64_simd_available(void)
 }
 
 #else
-#define BZ_KIND "blocked"
+#define BZ_KIND "blockwise"
 
 static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
                              const uint8_t *pr, const uint8_t *pa,
                              size_t n, uint64_t *sum_io, uint64_t mask)
 {
-    return bz64_scan_blocked(T, Trot, pr, pa, n, sum_io, mask);
+    return bz64_scan_blockwise(T, Trot, pr, pa, n, sum_io, mask);
 }
 
 static int bz64_simd_available(void)
@@ -336,7 +336,7 @@ static int bz64_simd_available(void)
 
 /* --- dispatch ----------------------------------------------------------- */
 
-/* resolved once (0 = blocked, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined);
+/* resolved once (0 = blockwise, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined);
  * a racy double-init writes the same value, so it is benign */
 static int bz64_use_simd = -1;
 
@@ -354,7 +354,7 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
 #endif
     if (bz64_use_simd)
         return bz64_scan_simd(table, table_rot, p_rem, p_add, n, sum, mask);
-    return bz64_scan_blocked(table, table_rot, p_rem, p_add, n, sum, mask);
+    return bz64_scan_blockwise(table, table_rot, p_rem, p_add, n, sum, mask);
 }
 
 const char *bz64_kernel_name(int force_scalar)
@@ -367,5 +367,5 @@ const char *bz64_kernel_name(int force_scalar)
     if (bz64_use_simd == 2)
         return BZ_KIND_512;
 #endif
-    return bz64_use_simd ? BZ_KIND : "blocked";
+    return bz64_use_simd ? BZ_KIND : "blockwise";
 }
diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h
index 2a9220cc73..032d5367c7 100644
--- a/src/borg/chunkers/buzhash64_impl.h
+++ b/src/borg/chunkers/buzhash64_impl.h
@@ -1,5 +1,5 @@
 /* buzhash64 chunker scan kernel: the cyclic-polynomial rolling hash inner
- * loop, with blocked/SIMD implementations that are bit-identical to the
+ * loop, with blockwise/SIMD implementations that are bit-identical to the
  * plain sequential loop (see buzhash64_impl.c for the algebra).
  *
  * The cut decision granularity stays at 1 byte: every position is tested
@@ -27,7 +27,8 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                  const uint8_t *p_rem, const uint8_t *p_add,
                  size_t n, uint64_t *sum, uint64_t mask, int force_scalar);
 
-/* Name of the kernel bz64_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */
+/* Name of the kernel bz64_scan would use: "avx512", "avx2", "blockwise" or "scalar"
+ * (there is no vector kernel on aarch64, see buzhash64_impl.c). */
 const char *bz64_kernel_name(int force_scalar);
 
 #endif
diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx
index 8d915d59c1..04f6414cb2 100644
--- a/src/borg/chunkers/fastcdc.pyx
+++ b/src/borg/chunkers/fastcdc.pyx
@@ -33,11 +33,11 @@ cdef extern from "fastcdc_impl.h":
 # class only provides the keyed Gear table and the _scan() hook calling the C kernel.
 #
 # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on
-# aarch64, AVX-512 or AVX2 on x86-64, blocked scalar elsewhere) that are bit-identical to
+# aarch64, AVX-512 or AVX2 on x86-64, blockwise scalar elsewhere) that are bit-identical to
 # the plain sequential Gear loop: every byte position is tested, cuts and hash state are
 # exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop
 # (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 and BORG_FASTCDC_NO_AVX2=1
-# at the blocked scalar kernel (for benchmarking, must be set before the first chunker use).
+# at the blockwise scalar kernel (for benchmarking, must be set before the first chunker use).
 
 
 @cython.boundscheck(False)
@@ -87,7 +87,7 @@ cdef class ChunkerFastCDC(ChunkerBase):
 
     @property
     def kernel(self):
-        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blocked' or 'scalar'."""
+        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'."""
         return (<bytes>fc_kernel_name(self.force_scalar)).decode("ascii")
 
     cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept:
diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c
index fe2c49a8d3..59e97457dd 100644
--- a/src/borg/chunkers/fastcdc_impl.c
+++ b/src/borg/chunkers/fastcdc_impl.c
@@ -21,7 +21,7 @@
  * which an exact sequential recheck of the block resolves. The per-lane
  * masks (mask << 7 .. mask << 0) are constants of the scan.
  *
- * All kernels (sequential, blocked scalar, NEON, AVX2, AVX-512) return
+ * All kernels (sequential, blockwise scalar, NEON, AVX2, AVX-512) return
  * bit-identical cut positions and fp values; the chunker's golden tests
  * depend on this. */
 
@@ -76,9 +76,9 @@ static inline void fc_block_prefix(const uint64_t *gear, const uint8_t *p, uint6
     s[7] = s8;
 }
 
-/* --- blocked scalar (portable C, no intrinsics) ------------------------- */
+/* --- blockwise scalar (portable C, no intrinsics) ------------------------- */
 
-static int64_t fc_scan_blocked(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
+static int64_t fc_scan_blockwise(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
 {
     uint64_t fp = *fp_io;
     uint64_t M[8];
@@ -215,7 +215,7 @@ static inline void fc_block_gear(const uint64_t *gear, const uint8_t *p, uint64_
  * forwarding and stalls, twice per block. Loading block i+1 while block i is
  * tested puts a full loop body between the stores and the load, so the stores
  * have retired by then and the stall disappears. Before this, the stalls made
- * the AVX2 kernel slower than the blocked scalar one. */
+ * the AVX2 kernel slower than the blockwise scalar one. */
 __attribute__((target("avx2"))) static int64_t
 fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
 {
@@ -351,7 +351,7 @@ static int fc_env_set(const char *name)
 static int fc_simd_available(void)
 {
     /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2, BORG_FASTCDC_NO_AVX2 at
-     * the blocked scalar kernel (for benchmarking the kernels against each
+     * the blockwise scalar kernel (for benchmarking the kernels against each
      * other); like the detection itself they are read once per process, so
      * they must be set before the first chunker use. */
     if (fc_env_set("BORG_FASTCDC_NO_AVX2"))
@@ -362,11 +362,11 @@ static int fc_simd_available(void)
 }
 
 #else
-#define FC_KIND "blocked"
+#define FC_KIND "blockwise"
 
 static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_io, uint64_t mask)
 {
-    return fc_scan_blocked(gear, p, n, fp_io, mask);
+    return fc_scan_blockwise(gear, p, n, fp_io, mask);
 }
 
 static int fc_simd_available(void)
@@ -378,7 +378,7 @@ static int fc_simd_available(void)
 
 /* --- dispatch ----------------------------------------------------------- */
 
-/* resolved once (0 = blocked, 1 = FC_KIND, 2 = FC_KIND_512 where defined);
+/* resolved once (0 = blockwise, 1 = FC_KIND, 2 = FC_KIND_512 where defined);
  * a racy double-init writes the same value, so it is benign */
 static int fc_use_simd = -1;
 
@@ -394,7 +394,7 @@ int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp,
 #endif
     if (fc_use_simd)
         return fc_scan_simd(gear, p, n, fp, mask);
-    return fc_scan_blocked(gear, p, n, fp, mask);
+    return fc_scan_blockwise(gear, p, n, fp, mask);
 }
 
 const char *fc_kernel_name(int force_scalar)
@@ -407,5 +407,5 @@ const char *fc_kernel_name(int force_scalar)
     if (fc_use_simd == 2)
         return FC_KIND_512;
 #endif
-    return fc_use_simd ? FC_KIND : "blocked";
+    return fc_use_simd ? FC_KIND : "blockwise";
 }
diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h
index a66606c707..eab87de6da 100644
--- a/src/borg/chunkers/fastcdc_impl.h
+++ b/src/borg/chunkers/fastcdc_impl.h
@@ -20,7 +20,7 @@
  * All kernels return bit-identical results. */
 int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar);
 
-/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blocked" or "scalar". */
+/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blockwise" or "scalar". */
 const char *fc_kernel_name(int force_scalar);
 
 #endif
diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py
index ebb9849475..3a336f80db 100644
--- a/src/borg/testsuite/chunkers/buzhash64_test.py
+++ b/src/borg/testsuite/chunkers/buzhash64_test.py
@@ -143,7 +143,7 @@ def rnd_key():
 
 
 def test_buzhash64_kernels_identical():
-    # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the plain sequential
+    # the SIMD scan kernel (avx512/avx2/blockwise, auto-selected) and the plain sequential
     # loop must produce identical cut points.
     data = os.urandom(4 * 1024 * 1024)
     key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
@@ -162,4 +162,4 @@ def sizes(chunker):
         del os.environ["BORG_BUZHASH64_FORCE_SCALAR"]
     assert sizes_default == sizes_scalar
     # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar")
+    assert default.kernel in ("avx512", "avx2", "blockwise", "scalar")
diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py
index d2ce69f3e0..4962bf99eb 100644
--- a/src/borg/testsuite/chunkers/fastcdc_test.py
+++ b/src/borg/testsuite/chunkers/fastcdc_test.py
@@ -179,7 +179,7 @@ def rnd_key():
 
 
 def test_fastcdc_kernels_identical():
-    # the SIMD scan kernel (neon/avx512/avx2/blocked, auto-selected) and the
+    # the SIMD scan kernel (neon/avx512/avx2/blockwise, auto-selected) and the
     # plain sequential Gear loop must produce identical cut points.
     data = os.urandom(4 * 1024 * 1024)
     key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
@@ -198,4 +198,4 @@ def sizes(chunker):
         del os.environ["BORG_FASTCDC_FORCE_SCALAR"]
     assert sizes_default == sizes_scalar
     # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("neon", "avx512", "avx2", "blocked", "scalar")
+    assert default.kernel in ("neon", "avx512", "avx2", "blockwise", "scalar")

From 350748c3e42c6614459d9d2cfb3b77920c65555b Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 13:21:07 +0200
Subject: [PATCH 036/144] chunkers: one BORG_*_KERNEL env var per chunker
 family

Replaces ten env vars with three:

  BORG_FASTCDC_NO_AVX512=1      ->  BORG_FASTCDC_KERNEL=avx2
  BORG_FASTCDC_NO_AVX2=1        ->  BORG_FASTCDC_KERNEL=blockwise
  BORG_FASTCDC_FORCE_SCALAR=1   ->  BORG_FASTCDC_KERNEL=scalar
  (same for BORG_BUZHASH64_*)
  BORG_PHTE_NO_VAES=1           ->  BORG_AES_CHUNKER_KERNEL=aes-ni
  BORG_<X>_AES_FORCE_EVP=1      ->  BORG_AES_CHUNKER_KERNEL=evp

The NO_* names were negative flags that did not do what they said: they
capped a dispatch ladder rather than disabling one kernel, so NO_AVX2=1
also disabled AVX-512. They could not express "use AVX-512" at all, they
used a second grammar (FORCE_*) for the same job, and they left no way
to select the blockwise kernel on aarch64, where the NO_AVX* vars are
no-ops.

The values are exactly the strings the .kernel property reports, so what
you read back is what you set. "auto" (the default) takes the best kernel
the CPU can run; any other value is a demand, and creating the chunker
raises ValueError if it cannot be honoured. Silently falling back is the
failure this is meant to prevent: it turns a benchmark, or a CI job that
means to pin one kernel, into a measurement of a different one. The error
separates the three reasons, which need different fixes - a typo, too old
a compiler, or the wrong CPU - because on x86-64 "your CPU has VAES but
your build does not" is a real and distinct case (it needs gcc >= 11 /
clang >= 14).

The three per-chunker *_AES_FORCE_EVP vars become one: all three AES
chunkers share phte_scan.h, so which scan paths exist is a property of
the build and the CPU, never of the individual chunker.

Selection now happens per chunker instance, so the old caveat that the
NO_* vars had to be set before the first chunker use is gone. Parsing and
validation live in one place (kernel_env.py plus the C name resolvers)
instead of being split between getenv() in C and os.environ in Cython.

None of the old names were ever released (they postdate 2.0.0b22).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/buzhash64.pyx               |  39 ++++--
 src/borg/chunkers/buzhash64_impl.c            | 112 ++++++++++++-----
 src/borg/chunkers/buzhash64_impl.h            |  32 ++++-
 src/borg/chunkers/fastcdc.pyx                 |  42 +++++--
 src/borg/chunkers/fastcdc_impl.c              | 119 +++++++++++++-----
 src/borg/chunkers/fastcdc_impl.h              |  36 +++++-
 src/borg/chunkers/goldilocks_aes.pyx          |  35 +++++-
 src/borg/chunkers/goldilocks_aes_impl.c       |   4 +-
 src/borg/chunkers/goldilocks_aes_impl.h       |   6 +-
 src/borg/chunkers/kernel_env.py               |  42 +++++++
 src/borg/chunkers/phte_core.h                 |  71 +++++++++--
 src/borg/chunkers/phte_kernel.h               |  36 ++++++
 src/borg/chunkers/rabin_aes.pyx               |  31 ++++-
 src/borg/chunkers/rabin_aes_impl.c            |   4 +-
 src/borg/chunkers/rabin_aes_impl.h            |   6 +-
 src/borg/chunkers/toeplitz_aes.pyx            |  33 ++++-
 src/borg/chunkers/toeplitz_aes_impl.c         |   4 +-
 src/borg/chunkers/toeplitz_aes_impl.h         |   6 +-
 src/borg/testsuite/chunkers/buzhash64_test.py |   4 +-
 src/borg/testsuite/chunkers/fastcdc_test.py   |  33 ++++-
 .../testsuite/chunkers/phte_chunkers_test.py  |  29 +++--
 21 files changed, 584 insertions(+), 140 deletions(-)
 create mode 100644 src/borg/chunkers/kernel_env.py
 create mode 100644 src/borg/chunkers/phte_kernel.h

diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx
index 03874b92c9..26c94de41e 100644
--- a/src/borg/chunkers/buzhash64.pyx
+++ b/src/borg/chunkers/buzhash64.pyx
@@ -17,8 +17,13 @@ from .base cimport ChunkerBase
 cdef extern from "buzhash64_impl.h":
     size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                      const uint8_t *p_rem, const uint8_t *p_add,
-                     size_t n, uint64_t *sum, uint64_t mask, int force_scalar) nogil
-    const char *bz64_kernel_name(int force_scalar)
+                     size_t n, uint64_t *sum, uint64_t mask, int kernel) nogil
+    const char *bz64_kernel_name(int kernel)
+    int bz64_kernel_select(const char *name, int *out_id)
+    const char *bz64_kernel_names()
+    int BZ_K_AUTO
+
+from .kernel_env import kernel_error, requested_kernel
 
 # Cyclic polynomial / buzhash
 #
@@ -101,6 +106,20 @@ cdef uint64_t _buzhash64_update(uint64_t sum, unsigned char remove, unsigned cha
     return BARREL_SHIFT64(sum, 1) ^ BARREL_SHIFT64(h[remove], lenmod) ^ h[add]
 
 
+cdef int _select_kernel() except -1:
+    """Resolve BORG_BUZHASH64_KERNEL to a kernel id, raising if it cannot be honoured."""
+    cdef int kid = BZ_K_AUTO
+    cdef int rc
+    want = requested_kernel("BORG_BUZHASH64_KERNEL")
+    if want is None:
+        return BZ_K_AUTO
+    rc = bz64_kernel_select(want.encode("ascii"), &kid)
+    if rc != 0:
+        raise kernel_error("BORG_BUZHASH64_KERNEL", want, rc,
+                           (<bytes>bz64_kernel_names()).decode("ascii"))
+    return kid
+
+
 cdef class ChunkerBuzHash64(ChunkerBase):
     """
     Content-Defined Chunker, variable chunk sizes.
@@ -118,7 +137,7 @@ cdef class ChunkerBuzHash64(ChunkerBase):
     """
     cdef uint64_t* table
     cdef uint64_t* table_rot
-    cdef int force_scalar
+    cdef int kernel_id
     cdef size_t window_size
 
     def __cinit__(self, bytes key, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int hash_window_size, int nc_level=0, size_t normal_size=0, bint sparse=False):
@@ -139,7 +158,7 @@ cdef class ChunkerBuzHash64(ChunkerBase):
         lenmod = <uint64_t>hash_window_size & 0x3f
         for i_rot in range(256):
             self.table_rot[i_rot] = BARREL_SHIFT64(self.table[i_rot], lenmod)
-        self.force_scalar = 1 if os.environ.get("BORG_BUZHASH64_FORCE_SCALAR", "") not in ("", "0") else 0
+        self.kernel_id = _select_kernel()
         # buzhash64 output is uniform, so contiguous low-bit masks are used (high_masks=False)
         self._setup_common("buzhash64", chunk_min_exp, chunk_max_exp, hash_mask_bits,
                            nc_level, normal_size, False, sparse)
@@ -155,8 +174,12 @@ cdef class ChunkerBuzHash64(ChunkerBase):
 
     @property
     def kernel(self):
-        """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'."""
-        return (<bytes>bz64_kernel_name(self.force_scalar)).decode("ascii")
+        """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'.
+
+        With BORG_BUZHASH64_KERNEL set to anything but "auto", this is always the
+        requested kernel - creating the chunker fails otherwise.
+        """
+        return (<bytes>bz64_kernel_name(self.kernel_id)).decode("ascii")
 
     cdef object process(self):
         """Process the chunker's buffer and return the next chunk."""
@@ -169,7 +192,7 @@ cdef class ChunkerBuzHash64(ChunkerBase):
         cdef uint8_t* stop_at
         cdef uint8_t* nc_stop
         cdef size_t did_bytes, span
-        cdef int force_scalar = self.force_scalar
+        cdef int kernel_id = self.kernel_id
 
         if self.done:
             if self.bytes_read == self.bytes_yielded:
@@ -229,7 +252,7 @@ cdef class ChunkerBuzHash64(ChunkerBase):
             span = stop_at - p
             with nogil:
                 did_bytes = bz64_scan(self.table, self.table_rot, p, p + window_size,
-                                      span, &sum, mask, force_scalar)
+                                      span, &sum, mask, kernel_id)
             self.position += did_bytes
             self.remaining -= did_bytes
 
diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index 27fdbcd0ad..4284a6d6d3 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -298,22 +298,12 @@ bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
-static int bz64_env_set(const char *name)
-{
-    const char *e = getenv(name);
-    return e != NULL && e[0] != 0 && strcmp(e, "0") != 0;
-}
-
 static int bz64_simd_available(void)
 {
-    /* BORG_BUZHASH64_NO_AVX512 caps dispatch at AVX2, BORG_BUZHASH64_NO_AVX2
-     * at the blockwise scalar kernel (for benchmarking the kernels against
-     * each other); like the detection itself they are read once per process,
-     * so they must be set before the first chunker use. */
-    if (bz64_env_set("BORG_BUZHASH64_NO_AVX2"))
-        return 0;
-    if (__builtin_cpu_supports("avx512f") && !bz64_env_set("BORG_BUZHASH64_NO_AVX512"))
+#ifdef BZ_KIND_512
+    if (__builtin_cpu_supports("avx512f"))
         return 2;
+#endif
     return __builtin_cpu_supports("avx2");
 }
 
@@ -334,38 +324,102 @@ static int bz64_simd_available(void)
 
 #endif
 
+/* --- kernel selection --------------------------------------------------- */
+
+const char *bz64_kernel_names(void)
+{
+#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+    return "auto, avx512, avx2, blockwise, scalar";
+#else
+    return "auto, blockwise, scalar";
+#endif
+}
+
+int bz64_kernel_select(const char *name, int *out_id)
+{
+    if (strcmp(name, "auto") == 0) {
+        *out_id = BZ_K_AUTO;
+        return BZ_KSEL_OK;
+    }
+    if (strcmp(name, "scalar") == 0) {
+        *out_id = BZ_K_SCALAR;
+        return BZ_KSEL_OK;
+    }
+    if (strcmp(name, "blockwise") == 0) {
+        *out_id = BZ_K_BLOCKWISE;
+        return BZ_KSEL_OK;
+    }
+#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+    if (strcmp(name, "avx2") == 0) {
+        if (!__builtin_cpu_supports("avx2"))
+            return BZ_KSEL_NOCPU;
+        *out_id = BZ_K_VECTOR;
+        return BZ_KSEL_OK;
+    }
+    if (strcmp(name, "avx512") == 0) {
+#ifndef BZ_KIND_512
+        return BZ_KSEL_NOTBUILT; /* compiler too old for the target attribute */
+#else
+        if (!__builtin_cpu_supports("avx512f"))
+            return BZ_KSEL_NOCPU;
+        *out_id = BZ_K_VECTOR512;
+        return BZ_KSEL_OK;
+#endif
+    }
+#endif
+    return BZ_KSEL_UNKNOWN;
+}
+
 /* --- dispatch ----------------------------------------------------------- */
 
-/* resolved once (0 = blockwise, 1 = BZ_KIND, 2 = BZ_KIND_512 where defined);
- * a racy double-init writes the same value, so it is benign */
-static int bz64_use_simd = -1;
+/* the kernel BZ_K_AUTO resolves to, worked out once; a racy double-init
+ * writes the same value, so it is benign */
+static int bz64_auto = -1;
+
+static int bz64_auto_kernel(void)
+{
+    int a;
+    if (bz64_auto < 0) {
+        a = bz64_simd_available();
+        bz64_auto = (a == 2) ? BZ_K_VECTOR512 : (a ? BZ_K_VECTOR : BZ_K_BLOCKWISE);
+    }
+    return bz64_auto;
+}
 
 size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                  const uint8_t *p_rem, const uint8_t *p_add,
-                 size_t n, uint64_t *sum, uint64_t mask, int force_scalar)
+                 size_t n, uint64_t *sum, uint64_t mask, int kernel)
 {
-    if (force_scalar)
+    if (kernel == BZ_K_AUTO)
+        kernel = bz64_auto_kernel();
+    switch (kernel) {
+    case BZ_K_SCALAR:
         return bz64_scan_seq(table, table_rot, p_rem, p_add, n, sum, mask);
-    if (bz64_use_simd < 0)
-        bz64_use_simd = bz64_simd_available();
+    case BZ_K_BLOCKWISE:
+        return bz64_scan_blockwise(table, table_rot, p_rem, p_add, n, sum, mask);
 #ifdef BZ_KIND_512
-    if (bz64_use_simd == 2)
+    case BZ_K_VECTOR512:
         return bz64_scan_simd512(table, table_rot, p_rem, p_add, n, sum, mask);
 #endif
-    if (bz64_use_simd)
+    default:
         return bz64_scan_simd(table, table_rot, p_rem, p_add, n, sum, mask);
-    return bz64_scan_blockwise(table, table_rot, p_rem, p_add, n, sum, mask);
+    }
 }
 
-const char *bz64_kernel_name(int force_scalar)
+const char *bz64_kernel_name(int kernel)
 {
-    if (force_scalar)
+    if (kernel == BZ_K_AUTO)
+        kernel = bz64_auto_kernel();
+    switch (kernel) {
+    case BZ_K_SCALAR:
         return "scalar";
-    if (bz64_use_simd < 0)
-        bz64_use_simd = bz64_simd_available();
+    case BZ_K_BLOCKWISE:
+        return "blockwise";
 #ifdef BZ_KIND_512
-    if (bz64_use_simd == 2)
+    case BZ_K_VECTOR512:
         return BZ_KIND_512;
 #endif
-    return bz64_use_simd ? BZ_KIND : "blockwise";
+    default:
+        return BZ_KIND;
+    }
 }
diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h
index 032d5367c7..f6dbfdd900 100644
--- a/src/borg/chunkers/buzhash64_impl.h
+++ b/src/borg/chunkers/buzhash64_impl.h
@@ -21,14 +21,34 @@
  * ROTL(table[b], window_size % 64) (precomputed by the caller).
  * p_rem points at the byte leaving the window, p_add at the byte entering
  * (p_add = p_rem + window_size); both must have n readable bytes.
- * force_scalar != 0 selects the sequential reference loop (for tests);
- * all kernels return bit-identical results. */
+ * kernel is one of BZ_K_*; BZ_K_AUTO picks the best one this CPU can run.
+ * All kernels return bit-identical results. */
+/* Scan kernel ids, a tier ladder; see fastcdc_impl.h for the rationale. */
+#define BZ_K_AUTO 0      /* best kernel this CPU can run */
+#define BZ_K_SCALAR 1    /* sequential reference loop */
+#define BZ_K_BLOCKWISE 2 /* portable 8-lane C */
+#define BZ_K_VECTOR 3    /* avx2 (there is no aarch64 vector kernel) */
+#define BZ_K_VECTOR512 4 /* avx512 */
+
+/* Results of bz64_kernel_select(). */
+#define BZ_KSEL_OK 0
+#define BZ_KSEL_UNKNOWN 1  /* not a kernel name on this platform */
+#define BZ_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */
+#define BZ_KSEL_NOCPU 3    /* known and built, but this CPU cannot run it */
+
+/* Resolve a kernel name for this build; see fc_kernel_select(). */
+int bz64_kernel_select(const char *name, int *out_id);
+
+/* Comma-separated list of the kernel names this build accepts. */
+const char *bz64_kernel_names(void);
+
 size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                  const uint8_t *p_rem, const uint8_t *p_add,
-                 size_t n, uint64_t *sum, uint64_t mask, int force_scalar);
+                 size_t n, uint64_t *sum, uint64_t mask, int kernel);
 
-/* Name of the kernel bz64_scan would use: "avx512", "avx2", "blockwise" or "scalar"
- * (there is no vector kernel on aarch64, see buzhash64_impl.c). */
-const char *bz64_kernel_name(int force_scalar);
+/* Name of the kernel <kernel> selects: "avx512", "avx2", "blockwise" or
+ * "scalar" (there is no vector kernel on aarch64, see buzhash64_impl.c).
+ * For BZ_K_AUTO, the auto-selected one. */
+const char *bz64_kernel_name(int kernel);
 
 #endif
diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx
index 04f6414cb2..ca0d11216c 100644
--- a/src/borg/chunkers/fastcdc.pyx
+++ b/src/borg/chunkers/fastcdc.pyx
@@ -13,8 +13,13 @@ from ..crypto.low_level import CSPRNG
 from .base cimport ChunkerBase
 
 cdef extern from "fastcdc_impl.h":
-    int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar) nogil
-    const char *fc_kernel_name(int force_scalar)
+    int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel) nogil
+    const char *fc_kernel_name(int kernel)
+    int fc_kernel_select(const char *name, int *out_id)
+    const char *fc_kernel_names()
+    int FC_K_AUTO
+
+from .kernel_env import kernel_error, requested_kernel
 
 # FastCDC content-defined chunker (Xia et al., USENIX ATC 2016).
 #
@@ -35,9 +40,8 @@ cdef extern from "fastcdc_impl.h":
 # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on
 # aarch64, AVX-512 or AVX2 on x86-64, blockwise scalar elsewhere) that are bit-identical to
 # the plain sequential Gear loop: every byte position is tested, cuts and hash state are
-# exactly the same, only faster. BORG_FASTCDC_FORCE_SCALAR=1 forces the sequential loop
-# (for tests); BORG_FASTCDC_NO_AVX512=1 caps dispatch at AVX2 and BORG_FASTCDC_NO_AVX2=1
-# at the blockwise scalar kernel (for benchmarking, must be set before the first chunker use).
+# exactly the same, only faster. BORG_FASTCDC_KERNEL pins the kernel instead of letting it
+# be auto-selected; see kernel_env.py and the .kernel property.
 
 
 @cython.boundscheck(False)
@@ -60,6 +64,20 @@ cdef uint64_t* fastcdc_init_gear(bytes key) except NULL:
     return gear
 
 
+cdef int _select_kernel() except -1:
+    """Resolve BORG_FASTCDC_KERNEL to a kernel id, raising if it cannot be honoured."""
+    cdef int kid = FC_K_AUTO
+    cdef int rc
+    want = requested_kernel("BORG_FASTCDC_KERNEL")
+    if want is None:
+        return FC_K_AUTO
+    rc = fc_kernel_select(want.encode("ascii"), &kid)
+    if rc != 0:
+        raise kernel_error("BORG_FASTCDC_KERNEL", want, rc,
+                           (<bytes>fc_kernel_names()).decode("ascii"))
+    return kid
+
+
 cdef class ChunkerFastCDC(ChunkerBase):
     """
     FastCDC content-defined chunker, variable chunk sizes, keyed Gear hash.
@@ -67,11 +85,11 @@ cdef class ChunkerFastCDC(ChunkerBase):
     Unlike the buzhash chunkers, Gear is window-less, so there is no hash_window_size parameter.
     """
     cdef uint64_t* gear
-    cdef int force_scalar
+    cdef int kernel_id
 
     def __cinit__(self, bytes key, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int nc_level=0, size_t normal_size=0, bint sparse=False):
         self.gear = NULL
-        self.force_scalar = 1 if os.environ.get("BORG_FASTCDC_FORCE_SCALAR", "") not in ("", "0") else 0
+        self.kernel_id = _select_kernel()
         self.gear = fastcdc_init_gear(key)
         # Gear accumulates information in its high bits, so the cut-decision
         # masks must use the high bits of the hash (high_masks=True). The Gear
@@ -87,13 +105,17 @@ cdef class ChunkerFastCDC(ChunkerBase):
 
     @property
     def kernel(self):
-        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'."""
-        return (<bytes>fc_kernel_name(self.force_scalar)).decode("ascii")
+        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'.
+
+        With BORG_FASTCDC_KERNEL set to anything but "auto", this is always the
+        requested kernel - creating the chunker fails otherwise.
+        """
+        return (<bytes>fc_kernel_name(self.kernel_id)).decode("ascii")
 
     cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept:
         cdef int64_t r
         with nogil:
-            r = fc_scan(self.gear, p, n, digest, mask, self.force_scalar)
+            r = fc_scan(self.gear, p, n, digest, mask, self.kernel_id)
         return r
 
 
diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c
index 59e97457dd..ac58923727 100644
--- a/src/borg/chunkers/fastcdc_impl.c
+++ b/src/borg/chunkers/fastcdc_impl.c
@@ -342,22 +342,12 @@ fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_i
     return -1;
 }
 
-static int fc_env_set(const char *name)
-{
-    const char *e = getenv(name);
-    return e != NULL && e[0] != 0 && strcmp(e, "0") != 0;
-}
-
 static int fc_simd_available(void)
 {
-    /* BORG_FASTCDC_NO_AVX512 caps dispatch at AVX2, BORG_FASTCDC_NO_AVX2 at
-     * the blockwise scalar kernel (for benchmarking the kernels against each
-     * other); like the detection itself they are read once per process, so
-     * they must be set before the first chunker use. */
-    if (fc_env_set("BORG_FASTCDC_NO_AVX2"))
-        return 0;
-    if (__builtin_cpu_supports("avx512f") && !fc_env_set("BORG_FASTCDC_NO_AVX512"))
+#ifdef FC_KIND_512
+    if (__builtin_cpu_supports("avx512f"))
         return 2;
+#endif
     return __builtin_cpu_supports("avx2");
 }
 
@@ -376,36 +366,107 @@ static int fc_simd_available(void)
 
 #endif
 
+/* --- kernel selection --------------------------------------------------- */
+
+const char *fc_kernel_names(void)
+{
+#if defined(__aarch64__)
+    return "auto, neon, blockwise, scalar";
+#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+    return "auto, avx512, avx2, blockwise, scalar";
+#else
+    return "auto, blockwise, scalar";
+#endif
+}
+
+int fc_kernel_select(const char *name, int *out_id)
+{
+    if (strcmp(name, "auto") == 0) {
+        *out_id = FC_K_AUTO;
+        return FC_KSEL_OK;
+    }
+    if (strcmp(name, "scalar") == 0) {
+        *out_id = FC_K_SCALAR;
+        return FC_KSEL_OK;
+    }
+    if (strcmp(name, "blockwise") == 0) {
+        *out_id = FC_K_BLOCKWISE;
+        return FC_KSEL_OK;
+    }
+#if defined(__aarch64__)
+    if (strcmp(name, "neon") == 0) {
+        *out_id = FC_K_VECTOR; /* baseline on aarch64, always runnable */
+        return FC_KSEL_OK;
+    }
+#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+    if (strcmp(name, "avx2") == 0) {
+        if (!__builtin_cpu_supports("avx2"))
+            return FC_KSEL_NOCPU;
+        *out_id = FC_K_VECTOR;
+        return FC_KSEL_OK;
+    }
+    if (strcmp(name, "avx512") == 0) {
+#ifndef FC_KIND_512
+        return FC_KSEL_NOTBUILT; /* compiler too old for the target attribute */
+#else
+        if (!__builtin_cpu_supports("avx512f"))
+            return FC_KSEL_NOCPU;
+        *out_id = FC_K_VECTOR512;
+        return FC_KSEL_OK;
+#endif
+    }
+#endif
+    return FC_KSEL_UNKNOWN;
+}
+
 /* --- dispatch ----------------------------------------------------------- */
 
-/* resolved once (0 = blockwise, 1 = FC_KIND, 2 = FC_KIND_512 where defined);
- * a racy double-init writes the same value, so it is benign */
-static int fc_use_simd = -1;
+/* the kernel FC_K_AUTO resolves to, worked out once; a racy double-init
+ * writes the same value, so it is benign */
+static int fc_auto = -1;
+
+static int fc_auto_kernel(void)
+{
+    int a;
+    if (fc_auto < 0) {
+        a = fc_simd_available();
+        fc_auto = (a == 2) ? FC_K_VECTOR512 : (a ? FC_K_VECTOR : FC_K_BLOCKWISE);
+    }
+    return fc_auto;
+}
 
-int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar)
+int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel)
 {
-    if (force_scalar)
+    if (kernel == FC_K_AUTO)
+        kernel = fc_auto_kernel();
+    switch (kernel) {
+    case FC_K_SCALAR:
         return fc_scan_seq(gear, p, n, fp, mask);
-    if (fc_use_simd < 0)
-        fc_use_simd = fc_simd_available();
+    case FC_K_BLOCKWISE:
+        return fc_scan_blockwise(gear, p, n, fp, mask);
 #ifdef FC_KIND_512
-    if (fc_use_simd == 2)
+    case FC_K_VECTOR512:
         return fc_scan_simd512(gear, p, n, fp, mask);
 #endif
-    if (fc_use_simd)
+    default:
         return fc_scan_simd(gear, p, n, fp, mask);
-    return fc_scan_blockwise(gear, p, n, fp, mask);
+    }
 }
 
-const char *fc_kernel_name(int force_scalar)
+const char *fc_kernel_name(int kernel)
 {
-    if (force_scalar)
+    if (kernel == FC_K_AUTO)
+        kernel = fc_auto_kernel();
+    switch (kernel) {
+    case FC_K_SCALAR:
         return "scalar";
-    if (fc_use_simd < 0)
-        fc_use_simd = fc_simd_available();
+    case FC_K_BLOCKWISE:
+        return "blockwise";
 #ifdef FC_KIND_512
-    if (fc_use_simd == 2)
+    case FC_K_VECTOR512:
         return FC_KIND_512;
 #endif
-    return fc_use_simd ? FC_KIND : "blockwise";
+    default:
+        return FC_KIND;
+    }
 }
diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h
index eab87de6da..145127b485 100644
--- a/src/borg/chunkers/fastcdc_impl.h
+++ b/src/borg/chunkers/fastcdc_impl.h
@@ -11,16 +11,42 @@
 #include <stdint.h>
 #include <stddef.h>
 
+/* Scan kernel ids, a tier ladder. Which names map onto them depends on the
+ * build: "neon" exists only on aarch64, "avx2"/"avx512" only on x86-64. */
+#define FC_K_AUTO 0      /* best kernel this CPU can run */
+#define FC_K_SCALAR 1    /* sequential reference loop */
+#define FC_K_BLOCKWISE 2 /* portable 8-lane C */
+#define FC_K_VECTOR 3    /* the platform's vector kernel: neon or avx2 */
+#define FC_K_VECTOR512 4 /* avx512 */
+
+/* Results of fc_kernel_select(). */
+#define FC_KSEL_OK 0
+#define FC_KSEL_UNKNOWN 1  /* not a kernel name on this platform */
+#define FC_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */
+#define FC_KSEL_NOCPU 3    /* known and built, but this CPU cannot run it */
+
+/* Resolve a kernel name for this build. On FC_KSEL_OK the id is stored in
+ * *out_id, otherwise *out_id is left alone. The three failures are kept apart
+ * because they need different fixes: a typo, too old a compiler, or the wrong
+ * CPU. */
+int fc_kernel_select(const char *name, int *out_id);
+
+/* Comma-separated list of the kernel names this build accepts, for error
+ * messages. Names a CPU cannot run are still listed. */
+const char *fc_kernel_names(void);
+
 /* Scan up to n positions: for i = 0..n-1 advance fp = (fp << 1) + gear[p[i]]
  * and test (fp & mask) == 0.
  * Returns the first i that matched (fp is left at position i), or -1 if none
  * matched (fp is left at position n-1).
- * gear: the keyed 256-entry table. force_scalar != 0 selects the sequential
- * reference loop (for tests); otherwise the best kernel for this CPU is used.
+ * gear: the keyed 256-entry table. kernel is one of FC_K_*; FC_K_AUTO picks
+ * the best one this CPU can run. Callers are expected to have validated any
+ * explicit choice with fc_kernel_select() - an unrunnable one falls back to
+ * FC_K_AUTO rather than crashing.
  * All kernels return bit-identical results. */
-int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int force_scalar);
+int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel);
 
-/* Name of the kernel fc_scan would use: "neon", "avx512", "avx2", "blockwise" or "scalar". */
-const char *fc_kernel_name(int force_scalar);
+/* Name of the kernel <kernel> selects; for FC_K_AUTO, the auto-selected one. */
+const char *fc_kernel_name(int kernel);
 
 #endif
diff --git a/src/borg/chunkers/goldilocks_aes.pyx b/src/borg/chunkers/goldilocks_aes.pyx
index 50a6e62d28..329577a723 100644
--- a/src/borg/chunkers/goldilocks_aes.pyx
+++ b/src/borg/chunkers/goldilocks_aes.pyx
@@ -45,7 +45,11 @@ cdef extern from "goldilocks_aes_impl.h":
     ctypedef struct GL_CTX:
         pass
     int GL_TABLES
-    GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t *aes_key, int force_sw)
+    GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t *aes_key, int kernel)
+    int phte_kernel_select(const char *name, int *out_id)
+    const char *phte_kernel_names()
+    int PHTE_K_AUTO
+    int PHTE_K_EVP
     void gl_free(GL_CTX *ctx)
     const char *gl_kind(const GL_CTX *ctx)
     uint64_t gl_digest64(const GL_CTX *ctx, const uint8_t *q)
@@ -105,6 +109,27 @@ def _derive(bytes key):
     return aes_key, k, tables
 
 
+from .kernel_env import kernel_error, requested_kernel
+
+
+cdef int _select_kernel() except -1:
+    """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured.
+
+    One selector for all three AES chunkers: they share phte_scan.h, so the
+    available paths never differ between them.
+    """
+    cdef int kid = PHTE_K_AUTO
+    cdef int rc
+    want = requested_kernel("BORG_AES_CHUNKER_KERNEL")
+    if want is None:
+        return PHTE_K_AUTO
+    rc = phte_kernel_select(want.encode("ascii"), &kid)
+    if rc != 0:
+        raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc,
+                           (<bytes>phte_kernel_names()).decode("ascii"))
+    return kid
+
+
 cdef class ChunkerGoldilocksAES(ChunkerPHTE):
     """
     Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision.
@@ -128,8 +153,8 @@ cdef class ChunkerGoldilocksAES(ChunkerPHTE):
             for i in range(256):
                 c_tables[t * 256 + i] = tables[t][i]
 
-        force_sw = os.environ.get("BORG_GOLDILOCKS_AES_FORCE_EVP", "") not in ("", "0")
-        self.ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, 1 if force_sw else 0)
+        kernel = _select_kernel()
+        self.ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, kernel)
         if self.ctx == NULL:
             raise MemoryError("Failed to set up goldilocks-aes kernel")
         self.kernel_str = (<bytes>gl_kind(self.ctx)).decode("ascii")
@@ -176,7 +201,7 @@ def goldilocks_aes_digest64(k, bytes window):
         for i in range(256):
             c_tables[t * 256 + i] = tables[t][i]
     memset(aes_key, 0, 16)
-    ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, 1)
+    ctx = gl_new(c_tables, k, (k * k) % _GL_P, aes_key, PHTE_K_EVP)
     if ctx == NULL:
         raise MemoryError("Failed to set up goldilocks-aes kernel")
     d = gl_digest64(ctx, <const uint8_t*>PyBytes_AsString(window))
@@ -205,7 +230,7 @@ def goldilocks_aes_scan_all(k, bytes aes_key, bytes data, mask, bint force_sw=Fa
     for t in range(len(tables)):
         for i in range(256):
             c_tables[t * 256 + i] = tables[t][i]
-    ctx = gl_new(c_tables, k, (k * k) % _GL_P, <const uint8_t*>PyBytes_AsString(aes_key), 1 if force_sw else 0)
+    ctx = gl_new(c_tables, k, (k * k) % _GL_P, <const uint8_t*>PyBytes_AsString(aes_key), PHTE_K_EVP if force_sw else PHTE_K_AUTO)
     if ctx == NULL:
         raise MemoryError("Failed to set up goldilocks-aes kernel")
     try:
diff --git a/src/borg/chunkers/goldilocks_aes_impl.c b/src/borg/chunkers/goldilocks_aes_impl.c
index f808a59ebe..f6760ce16f 100644
--- a/src/borg/chunkers/goldilocks_aes_impl.c
+++ b/src/borg/chunkers/goldilocks_aes_impl.c
@@ -135,7 +135,7 @@ uint64_t gl_digest64(const GL_CTX *c, const uint8_t *q)
 #include "phte_scan.h"
 
 GL_CTX *gl_new(const uint64_t tables[GL_TABLES * 256], uint64_t k1, uint64_t k2,
-               const uint8_t aes_key[16], int force_sw)
+               const uint8_t aes_key[16], int kernel)
 {
     GL_CTX *c = calloc(1, sizeof(GL_CTX));
     if (c == NULL)
@@ -145,7 +145,7 @@ GL_CTX *gl_new(const uint64_t tables[GL_TABLES * 256], uint64_t k1, uint64_t k2,
     memcpy(c->in1_tbl, tables + 2 * 256, sizeof(c->in1_tbl));
     c->k1 = k1;
     c->k2 = k2;
-    if (!phte_base_init(&c->base, aes_key, force_sw)) {
+    if (!phte_base_init(&c->base, aes_key, kernel)) {
         free(c);
         return NULL;
     }
diff --git a/src/borg/chunkers/goldilocks_aes_impl.h b/src/borg/chunkers/goldilocks_aes_impl.h
index e758cae44a..f1f04a3444 100644
--- a/src/borg/chunkers/goldilocks_aes_impl.h
+++ b/src/borg/chunkers/goldilocks_aes_impl.h
@@ -21,6 +21,8 @@
 #include <stdint.h>
 #include <stddef.h>
 
+#include "phte_kernel.h"
+
 typedef struct GL_CTX GL_CTX;
 
 /* Number of 256-entry rolling tables passed to gl_new, in this order:
@@ -37,9 +39,9 @@ typedef struct GL_CTX GL_CTX;
  * k1: the secret evaluation point K (canonical, 0 <= K < p).
  * k2: K^2 mod p.
  * aes_key: 16 bytes (AES-128).
- * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks).
+ * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
  * Returns NULL on allocation/OpenSSL failure. */
-GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t aes_key[16], int force_sw);
+GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t aes_key[16], int kernel);
 
 void gl_free(GL_CTX *ctx);
 
diff --git a/src/borg/chunkers/kernel_env.py b/src/borg/chunkers/kernel_env.py
new file mode 100644
index 0000000000..aacfcc76ae
--- /dev/null
+++ b/src/borg/chunkers/kernel_env.py
@@ -0,0 +1,42 @@
+"""Shared parsing for the BORG_*_KERNEL scan-kernel selection env vars.
+
+Each SIMD-accelerated chunker picks its inner scan kernel automatically. The
+env vars exist to pin it instead - for benchmarking one kernel against
+another, and for CI that wants to prove a particular kernel was exercised
+rather than hope it was.
+
+"auto" (the default) takes the best kernel the CPU can run. Any other value is
+a demand, not a preference: if that kernel cannot run here, chunker creation
+raises ValueError rather than quietly falling back, because a silent fallback
+turns a benchmark or a test into a measurement of something else.
+
+The C side resolves names, since which names exist depends on the build and
+the CPU; this module only turns its verdict into an exception.
+"""
+
+import os
+
+# must match the FC_KSEL_* / BZ_KSEL_* / PHTE_KSEL_* codes in the C headers
+KSEL_OK = 0
+KSEL_UNKNOWN = 1
+KSEL_NOTBUILT = 2
+KSEL_NOCPU = 3
+
+
+def kernel_error(envvar, want, status, valid):
+    """The ValueError for a kernel request that cannot be honoured."""
+    if status == KSEL_UNKNOWN:
+        detail = "not a kernel of this build"
+    elif status == KSEL_NOTBUILT:
+        detail = "not compiled into this build (needs a newer compiler)"
+    elif status == KSEL_NOCPU:
+        detail = "this CPU does not support it"
+    else:
+        detail = "unavailable"
+    return ValueError(f"{envvar}={want!r}: {detail}. Valid values: {valid}")
+
+
+def requested_kernel(envvar):
+    """The kernel name requested via <envvar>, or None for auto/unset."""
+    want = os.environ.get(envvar, "").strip().lower()
+    return None if want in ("", "auto") else want
diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h
index ec8ad6a2e4..7c7925e698 100644
--- a/src/borg/chunkers/phte_core.h
+++ b/src/borg/chunkers/phte_core.h
@@ -13,6 +13,8 @@
 #ifndef BORG_PHTE_CORE_H
 #define BORG_PHTE_CORE_H
 
+#include "phte_kernel.h"
+
 #include <stdint.h>
 #include <stddef.h>
 #include <stdlib.h>
@@ -141,14 +143,7 @@ phte_aes1_ni(const uint8_t rk[11][16], __m128i b)
 
 static int phte_hw512_available(void)
 {
-    /* BORG_PHTE_NO_VAES caps the AES chunkers at the 128-bit AES-NI path
-     * (for benchmarking the paths against each other); it is read at
-     * context creation, ""/"0" mean off. */
-    const char *e;
-    if (!__builtin_cpu_supports("avx512f") || !__builtin_cpu_supports("vaes"))
-        return 0;
-    e = getenv("BORG_PHTE_NO_VAES");
-    return e == NULL || e[0] == 0 || strcmp(e, "0") == 0;
+    return __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("vaes");
 }
 #endif
 
@@ -160,24 +155,76 @@ static int phte_hw512_available(void)
 #define PHTE_HAVE_HW512 0
 #endif
 
+/* --- kernel selection ---------------------------------------------------
+ *
+ * Shared by all three AES chunkers: the scan path lives in phte_scan.h and
+ * only the rolling hash differs between them, so there is no machine on which
+ * one of them has a hardware path and another does not. Hence one selector
+ * (BORG_AES_CHUNKER_KERNEL) rather than one per chunker. */
+
+const char *phte_kernel_names(void)
+{
+#if defined(__aarch64__)
+    return "auto, aes-arm64, evp";
+#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+    return "auto, vaes, aes-ni, evp";
+#else
+    return "auto, evp";
+#endif
+}
+
+int phte_kernel_select(const char *name, int *out_id)
+{
+    if (strcmp(name, "auto") == 0) {
+        *out_id = PHTE_K_AUTO;
+        return PHTE_KSEL_OK;
+    }
+    if (strcmp(name, "evp") == 0) {
+        *out_id = PHTE_K_EVP;
+        return PHTE_KSEL_OK;
+    }
+#if PHTE_HAVE_HW
+    if (strcmp(name, PHTE_KIND_HW) == 0) {
+        if (!phte_hw_available())
+            return PHTE_KSEL_NOCPU;
+        *out_id = PHTE_K_HW;
+        return PHTE_KSEL_OK;
+    }
+#endif
+#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+    if (strcmp(name, "vaes") == 0) {
+#if !PHTE_HAVE_HW512
+        return PHTE_KSEL_NOTBUILT; /* compiler too old for the VAES intrinsics */
+#else
+        if (!phte_hw512_available())
+            return PHTE_KSEL_NOCPU;
+        *out_id = PHTE_K_HW512;
+        return PHTE_KSEL_OK;
+#endif
+    }
+#endif
+    return PHTE_KSEL_UNKNOWN;
+}
+
 /* --- context base management ------------------------------------------- */
 
 /* Expand the AES key, select the scan path and set up the OpenSSL context.
+ * kernel is one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
  * Returns 0 on failure (caller frees its context). */
-static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int force_sw)
+static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int kernel)
 {
     phte_aes128_expand(aes_key, b->rk);
 #if PHTE_HAVE_HW
-    b->use_hw = !force_sw && phte_hw_available();
+    b->use_hw = (kernel == PHTE_K_AUTO || kernel == PHTE_K_HW || kernel == PHTE_K_HW512) && phte_hw_available();
 #else
-    (void)force_sw;
     b->use_hw = 0;
 #endif
 #if PHTE_HAVE_HW512
-    b->use_hw512 = b->use_hw && phte_hw512_available();
+    b->use_hw512 = b->use_hw && kernel != PHTE_K_HW && phte_hw512_available();
 #else
     b->use_hw512 = 0;
 #endif
+    (void)kernel;
     b->evp = EVP_CIPHER_CTX_new();
     if (b->evp == NULL)
         return 0;
diff --git a/src/borg/chunkers/phte_kernel.h b/src/borg/chunkers/phte_kernel.h
new file mode 100644
index 0000000000..1340c55d34
--- /dev/null
+++ b/src/borg/chunkers/phte_kernel.h
@@ -0,0 +1,36 @@
+/* Scan-path selection shared by the three UHF-then-PRF ("Chk-PHTE") chunkers
+ * (toeplitz-aes, rabin-aes, goldilocks-aes).
+ *
+ * They share phte_scan.h and differ only in their rolling hash, so the
+ * available scan paths are a property of the build and the CPU, never of the
+ * individual chunker: there is no machine where one of them has a hardware
+ * path and another does not. Hence one selector for all three. */
+
+#ifndef BORG_PHTE_KERNEL_H
+#define BORG_PHTE_KERNEL_H
+
+/* Scan path ids, a tier ladder. Which names map onto them depends on the
+ * build: "vaes" only on x86-64, and the 128-bit hardware path is spelled
+ * "aes-ni" on x86-64 and "aes-arm64" on aarch64. */
+#define PHTE_K_AUTO 0  /* best path this CPU can run */
+#define PHTE_K_EVP 1   /* portable OpenSSL EVP batch path */
+#define PHTE_K_HW 2    /* 128-bit AES instructions: aes-ni / aes-arm64 */
+#define PHTE_K_HW512 3 /* VAES/AVX-512, 4 AES blocks per instruction */
+
+/* Results of phte_kernel_select(). */
+#define PHTE_KSEL_OK 0
+#define PHTE_KSEL_UNKNOWN 1  /* not a scan path name on this platform */
+#define PHTE_KSEL_NOTBUILT 2 /* known, but not compiled into this binary */
+#define PHTE_KSEL_NOCPU 3    /* known and built, but this CPU cannot run it */
+
+/* Resolve a scan path name for this build. On PHTE_KSEL_OK the id is stored
+ * in *out_id, otherwise *out_id is left alone. The three failures are kept
+ * apart because they need different fixes: a typo, too old a compiler, or the
+ * wrong CPU. */
+int phte_kernel_select(const char *name, int *out_id);
+
+/* Comma-separated list of the scan path names this build accepts, for error
+ * messages. Names a CPU cannot run are still listed. */
+const char *phte_kernel_names(void);
+
+#endif /* BORG_PHTE_KERNEL_H */
diff --git a/src/borg/chunkers/rabin_aes.pyx b/src/borg/chunkers/rabin_aes.pyx
index 8198a8f17b..fa55ded145 100644
--- a/src/borg/chunkers/rabin_aes.pyx
+++ b/src/borg/chunkers/rabin_aes.pyx
@@ -57,7 +57,11 @@ cdef extern from "rabin_aes_impl.h":
     ctypedef struct RA_CTX:
         pass
     int RA_TABLES
-    RA_CTX *ra_new(const uint64_t *tables, const uint8_t *aes_key, int force_sw)
+    RA_CTX *ra_new(const uint64_t *tables, const uint8_t *aes_key, int kernel)
+    int phte_kernel_select(const char *name, int *out_id)
+    const char *phte_kernel_names()
+    int PHTE_K_AUTO
+    int PHTE_K_EVP
     void ra_free(RA_CTX *ctx)
     const char *ra_kind(const RA_CTX *ctx)
     uint64_t ra_digest64(const RA_CTX *ctx, const uint8_t *q)
@@ -175,6 +179,27 @@ def _derive(bytes key):
     return aes_key, p, tables
 
 
+from .kernel_env import kernel_error, requested_kernel
+
+
+cdef int _select_kernel() except -1:
+    """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured.
+
+    One selector for all three AES chunkers: they share phte_scan.h, so the
+    available paths never differ between them.
+    """
+    cdef int kid = PHTE_K_AUTO
+    cdef int rc
+    want = requested_kernel("BORG_AES_CHUNKER_KERNEL")
+    if want is None:
+        return PHTE_K_AUTO
+    rc = phte_kernel_select(want.encode("ascii"), &kid)
+    if rc != 0:
+        raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc,
+                           (<bytes>phte_kernel_names()).decode("ascii"))
+    return kid
+
+
 cdef class ChunkerRabinAES(ChunkerPHTE):
     """
     Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision.
@@ -198,8 +223,8 @@ cdef class ChunkerRabinAES(ChunkerPHTE):
             for i in range(256):
                 c_tables[t * 256 + i] = tables[t][i]
 
-        force_sw = os.environ.get("BORG_RABIN_AES_FORCE_EVP", "") not in ("", "0")
-        self.ctx = ra_new(c_tables, aes_key, 1 if force_sw else 0)
+        kernel = _select_kernel()
+        self.ctx = ra_new(c_tables, aes_key, kernel)
         if self.ctx == NULL:
             raise MemoryError("Failed to set up rabin-aes kernel")
         self.kernel_str = (<bytes>ra_kind(self.ctx)).decode("ascii")
diff --git a/src/borg/chunkers/rabin_aes_impl.c b/src/borg/chunkers/rabin_aes_impl.c
index fc21611f8f..af3ab6d1b1 100644
--- a/src/borg/chunkers/rabin_aes_impl.c
+++ b/src/borg/chunkers/rabin_aes_impl.c
@@ -66,7 +66,7 @@ uint64_t ra_digest64(const RA_CTX *c, const uint8_t *q)
 #define PH_DIGEST64 ra_digest64
 #include "phte_scan.h"
 
-RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16], int force_sw)
+RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16], int kernel)
 {
     RA_CTX *c = calloc(1, sizeof(RA_CTX));
     if (c == NULL)
@@ -76,7 +76,7 @@ RA_CTX *ra_new(const uint64_t tables[RA_TABLES * 256], const uint8_t aes_key[16]
     memcpy(c->w1_tbl, tables + 2 * 256, sizeof(c->w1_tbl));
     memcpy(c->out8_tbl, tables + 3 * 256, sizeof(c->out8_tbl));
     memcpy(c->out16_tbl, tables + 4 * 256, sizeof(c->out16_tbl));
-    if (!phte_base_init(&c->base, aes_key, force_sw)) {
+    if (!phte_base_init(&c->base, aes_key, kernel)) {
         free(c);
         return NULL;
     }
diff --git a/src/borg/chunkers/rabin_aes_impl.h b/src/borg/chunkers/rabin_aes_impl.h
index 22fe17ad46..5a4fc07e40 100644
--- a/src/borg/chunkers/rabin_aes_impl.h
+++ b/src/borg/chunkers/rabin_aes_impl.h
@@ -19,6 +19,8 @@
 #include <stdint.h>
 #include <stddef.h>
 
+#include "phte_kernel.h"
+
 typedef struct RA_CTX RA_CTX;
 
 /* Number of 256-entry rolling tables passed to ra_new, in this order:
@@ -34,9 +36,9 @@ typedef struct RA_CTX RA_CTX;
 /* Create a kernel context.
  * tables: RA_TABLES * 256 uint64 entries, see above.
  * aes_key: 16 bytes (AES-128).
- * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks).
+ * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
  * Returns NULL on allocation/OpenSSL failure. */
-RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw);
+RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel);
 
 void ra_free(RA_CTX *ctx);
 
diff --git a/src/borg/chunkers/toeplitz_aes.pyx b/src/borg/chunkers/toeplitz_aes.pyx
index 482ed4d3db..8bc5ff423c 100644
--- a/src/borg/chunkers/toeplitz_aes.pyx
+++ b/src/borg/chunkers/toeplitz_aes.pyx
@@ -50,7 +50,11 @@ cdef extern from "toeplitz_aes_impl.h":
     ctypedef struct TP_CTX:
         pass
     int TP_TABLES
-    TP_CTX *tp_new(const uint64_t *tables, const uint8_t *aes_key, int force_sw)
+    TP_CTX *tp_new(const uint64_t *tables, const uint8_t *aes_key, int kernel)
+    int phte_kernel_select(const char *name, int *out_id)
+    const char *phte_kernel_names()
+    int PHTE_K_AUTO
+    int PHTE_K_EVP
     void tp_free(TP_CTX *ctx)
     const char *tp_kind(const TP_CTX *ctx)
     uint64_t tp_digest64(const TP_CTX *ctx, const uint8_t *q)
@@ -107,6 +111,27 @@ def _derive(bytes key):
     return aes_key, t, tables
 
 
+from .kernel_env import kernel_error, requested_kernel
+
+
+cdef int _select_kernel() except -1:
+    """Resolve BORG_AES_CHUNKER_KERNEL to a scan path id, raising if it cannot be honoured.
+
+    One selector for all three AES chunkers: they share phte_scan.h, so the
+    available paths never differ between them.
+    """
+    cdef int kid = PHTE_K_AUTO
+    cdef int rc
+    want = requested_kernel("BORG_AES_CHUNKER_KERNEL")
+    if want is None:
+        return PHTE_K_AUTO
+    rc = phte_kernel_select(want.encode("ascii"), &kid)
+    if rc != 0:
+        raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc,
+                           (<bytes>phte_kernel_names()).decode("ascii"))
+    return kid
+
+
 cdef class ChunkerToeplitzAES(ChunkerPHTE):
     """
     Content-Defined Chunker, variable chunk sizes, UHF-then-PRF cut decision.
@@ -130,8 +155,8 @@ cdef class ChunkerToeplitzAES(ChunkerPHTE):
             for i in range(256):
                 c_tables[t * 256 + i] = tables[t][i]
 
-        force_sw = os.environ.get("BORG_TOEPLITZ_AES_FORCE_EVP", "") not in ("", "0")
-        self.ctx = tp_new(c_tables, aes_key, 1 if force_sw else 0)
+        kernel = _select_kernel()
+        self.ctx = tp_new(c_tables, aes_key, kernel)
         if self.ctx == NULL:
             raise MemoryError("Failed to set up toeplitz-aes kernel")
         self.kernel_str = (<bytes>tp_kind(self.ctx)).decode("ascii")
@@ -177,7 +202,7 @@ def toeplitz_aes_digest64(bytes key, bytes window):
         for i in range(256):
             c_tables[t * 256 + i] = tables[t][i]
     memset(aes_key_c, 0, 16)
-    ctx = tp_new(c_tables, aes_key_c, 1)
+    ctx = tp_new(c_tables, aes_key_c, PHTE_K_EVP)
     if ctx == NULL:
         raise MemoryError("Failed to set up toeplitz-aes kernel")
     d = tp_digest64(ctx, <const uint8_t*>PyBytes_AsString(window))
diff --git a/src/borg/chunkers/toeplitz_aes_impl.c b/src/borg/chunkers/toeplitz_aes_impl.c
index 4e9239bddc..36797d60d3 100644
--- a/src/borg/chunkers/toeplitz_aes_impl.c
+++ b/src/borg/chunkers/toeplitz_aes_impl.c
@@ -84,7 +84,7 @@ uint64_t tp_digest64(const TP_CTX *c, const uint8_t *q)
 #define PH_DIGEST64 tp_digest64
 #include "phte_scan.h"
 
-TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16], int force_sw)
+TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16], int kernel)
 {
     TP_CTX *c = calloc(1, sizeof(TP_CTX));
     if (c == NULL)
@@ -93,7 +93,7 @@ TP_CTX *tp_new(const uint64_t tables[TP_TABLES * 256], const uint8_t aes_key[16]
     memcpy(c->in1_tbl, tables + 1 * 256, sizeof(c->in1_tbl));
     memcpy(c->out64_tbl, tables + 2 * 256, sizeof(c->out64_tbl));
     memcpy(c->out65_tbl, tables + 3 * 256, sizeof(c->out65_tbl));
-    if (!phte_base_init(&c->base, aes_key, force_sw)) {
+    if (!phte_base_init(&c->base, aes_key, kernel)) {
         free(c);
         return NULL;
     }
diff --git a/src/borg/chunkers/toeplitz_aes_impl.h b/src/borg/chunkers/toeplitz_aes_impl.h
index dec05d32e7..5aa3f61f8b 100644
--- a/src/borg/chunkers/toeplitz_aes_impl.h
+++ b/src/borg/chunkers/toeplitz_aes_impl.h
@@ -24,6 +24,8 @@
 #include <stdint.h>
 #include <stddef.h>
 
+#include "phte_kernel.h"
+
 typedef struct TP_CTX TP_CTX;
 
 /* Number of 256-entry rolling tables passed to tp_new, in this order:
@@ -38,9 +40,9 @@ typedef struct TP_CTX TP_CTX;
 /* Create a kernel context.
  * tables: TP_TABLES * 256 uint64 entries, see above.
  * aes_key: 16 bytes (AES-128).
- * force_sw: nonzero forces the portable OpenSSL path (for tests/benchmarks).
+ * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
  * Returns NULL on allocation/OpenSSL failure. */
-TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int force_sw);
+TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel);
 
 void tp_free(TP_CTX *ctx);
 
diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py
index 3a336f80db..eec877000b 100644
--- a/src/borg/testsuite/chunkers/buzhash64_test.py
+++ b/src/borg/testsuite/chunkers/buzhash64_test.py
@@ -153,13 +153,13 @@ def sizes(chunker):
 
     default = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
     sizes_default = sizes(default)
-    os.environ["BORG_BUZHASH64_FORCE_SCALAR"] = "1"
+    os.environ["BORG_BUZHASH64_KERNEL"] = "scalar"
     try:
         forced = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
         assert forced.kernel == "scalar"
         sizes_scalar = sizes(forced)
     finally:
-        del os.environ["BORG_BUZHASH64_FORCE_SCALAR"]
+        del os.environ["BORG_BUZHASH64_KERNEL"]
     assert sizes_default == sizes_scalar
     # whatever kernel was selected by default, it must be a known one
     assert default.kernel in ("avx512", "avx2", "blockwise", "scalar")
diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py
index 4962bf99eb..8d860c2ab5 100644
--- a/src/borg/testsuite/chunkers/fastcdc_test.py
+++ b/src/borg/testsuite/chunkers/fastcdc_test.py
@@ -189,13 +189,42 @@ def sizes(chunker):
 
     default = ChunkerFastCDC(key0, 10, 16, 14, 2)
     sizes_default = sizes(default)
-    os.environ["BORG_FASTCDC_FORCE_SCALAR"] = "1"
+    os.environ["BORG_FASTCDC_KERNEL"] = "scalar"
     try:
         forced = ChunkerFastCDC(key0, 10, 16, 14, 2)
         assert forced.kernel == "scalar"
         sizes_scalar = sizes(forced)
     finally:
-        del os.environ["BORG_FASTCDC_FORCE_SCALAR"]
+        del os.environ["BORG_FASTCDC_KERNEL"]
     assert sizes_default == sizes_scalar
     # whatever kernel was selected by default, it must be a known one
     assert default.kernel in ("neon", "avx512", "avx2", "blockwise", "scalar")
+
+
+@pytest.mark.parametrize("envvar", ["BORG_FASTCDC_KERNEL", "BORG_BUZHASH64_KERNEL", "BORG_AES_CHUNKER_KERNEL"])
+def test_kernel_env_rejects_unusable(envvar, monkeypatch):
+    # A kernel that cannot run here must fail loudly instead of silently
+    # falling back - a silent fallback would turn a benchmark, or a CI job that
+    # means to pin one kernel, into a measurement of a different one.
+    from ...chunkers import ChunkerBuzHash64, ChunkerToeplitzAES
+
+    make = {
+        "BORG_FASTCDC_KERNEL": lambda k: ChunkerFastCDC(k, 10, 16, 14, 2),
+        "BORG_BUZHASH64_KERNEL": lambda k: ChunkerBuzHash64(k, 10, 16, 14, 4095, 2),
+        "BORG_AES_CHUNKER_KERNEL": lambda k: ChunkerToeplitzAES(k, 10, 16, 14, 2),
+    }[envvar]
+    key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
+
+    monkeypatch.setenv(envvar, "no-such-kernel")
+    with pytest.raises(ValueError, match="no-such-kernel"):
+        make(key0)
+
+    # "auto" and an unset var both mean "pick the best one", and always work
+    monkeypatch.setenv(envvar, "auto")
+    auto = make(key0).kernel
+    monkeypatch.delenv(envvar)
+    assert make(key0).kernel == auto
+
+    # asking for whatever auto-selection picked must yield exactly that
+    monkeypatch.setenv(envvar, auto)
+    assert make(key0).kernel == auto
diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py
index 2dc2cac3e6..50b54220f5 100644
--- a/src/borg/testsuite/chunkers/phte_chunkers_test.py
+++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py
@@ -23,13 +23,16 @@
 # from os.urandom(32)
 key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
 
-# (chunker class, algo name, default params constant, env var forcing the portable kernel)
+# all three share one scan-path selector, see phte_kernel.h
+KERNEL_ENV = "BORG_AES_CHUNKER_KERNEL"
+
+# (chunker class, algo name, default params constant)
 CHUNKERS = [
-    (ChunkerRabinAES, CH_RABIN_AES, RABIN_AES_PARAMS, "BORG_RABIN_AES_FORCE_EVP"),
-    (ChunkerGoldilocksAES, CH_GOLDILOCKS_AES, GOLDILOCKS_AES_PARAMS, "BORG_GOLDILOCKS_AES_FORCE_EVP"),
-    (ChunkerToeplitzAES, CH_TOEPLITZ_AES, TOEPLITZ_AES_PARAMS, "BORG_TOEPLITZ_AES_FORCE_EVP"),
+    (ChunkerRabinAES, CH_RABIN_AES, RABIN_AES_PARAMS),
+    (ChunkerGoldilocksAES, CH_GOLDILOCKS_AES, GOLDILOCKS_AES_PARAMS),
+    (ChunkerToeplitzAES, CH_TOEPLITZ_AES, TOEPLITZ_AES_PARAMS),
 ]
-IDS = [algo for _, algo, _, _ in CHUNKERS]
+IDS = [algo for _, algo, _ in CHUNKERS]
 
 
 def H(data):
@@ -44,7 +47,7 @@ def chunker_spec(request):
 def test_kernels_identical(chunker_spec):
     # the OpenSSL EVP batch path and the AES hardware instruction path (if available
     # on this platform) must produce identical cut points.
-    cls, algo, params, env_var = chunker_spec
+    cls, algo, params = chunker_spec
     data = os.urandom(4 * 1024 * 1024)
 
     def sizes(chunker):
@@ -52,20 +55,20 @@ def sizes(chunker):
 
     default = cls(key0, 10, 16, 14, 2)
     sizes_default = sizes(default)
-    os.environ[env_var] = "1"
+    os.environ[KERNEL_ENV] = "evp"
     try:
         forced = cls(key0, 10, 16, 14, 2)
         assert forced.kernel == "evp"
         sizes_evp = sizes(forced)
     finally:
-        del os.environ[env_var]
+        del os.environ[KERNEL_ENV]
     assert sizes_default == sizes_evp
     # whatever kernel was selected by default, it must be a known one
     assert default.kernel in ("aes-arm64", "vaes", "aes-ni", "evp")
 
 
 def test_chunksize_distribution(chunker_spec):
-    cls, algo, params, env_var = chunker_spec
+    cls, algo, params = chunker_spec
     data = os.urandom(1048576)
     min_exp, max_exp, mask, nc_level = 10, 16, 14, 2  # chunk size target 16 KiB, clip at 1 KiB and 64 KiB
     chunker = cls(key0, min_exp, max_exp, mask, nc_level)
@@ -95,7 +98,7 @@ def test_chunksize_distribution(chunker_spec):
 def test_shift_resilience(chunker_spec):
     # content-defined cuts must survive a prefix insertion (this also validates that the
     # rolling digest update and the per-chunk window warm-up agree with each other).
-    cls, algo, params, env_var = chunker_spec
+    cls, algo, params = chunker_spec
     data = os.urandom(4 * 1024 * 1024)
 
     def chunk_hashes(data):
@@ -110,7 +113,7 @@ def chunk_hashes(data):
 
 def test_get_chunker(chunker_spec):
     # without a key, get_chunker uses an all-zero key; chunking must still work and be deterministic
-    cls, algo, params, env_var = chunker_spec
+    cls, algo, params = chunker_spec
     data = os.urandom(2 * 1024 * 1024)
     a = cf_expand(get_chunker(*params, key=None).chunkify(BytesIO(data)))
     b = cf_expand(get_chunker(algo, 19, 23, 21, 2, key=None).chunkify(BytesIO(data)))
@@ -123,7 +126,7 @@ def test_params_parsing(chunker_spec):
 
     from ...helpers import ChunkerParams
 
-    cls, algo, params, env_var = chunker_spec
+    cls, algo, params = chunker_spec
 
     # <algo>, chunk_min, chunk_max, chunk_mask, nc_level (no window field)
     assert ChunkerParams(f"{algo},19,23,21,2") == (algo, 19, 23, 21, 2)
@@ -149,7 +152,7 @@ def test_params_parsing(chunker_spec):
 @pytest.mark.parametrize("worker", range(os.cpu_count() or 1))
 def test_fuzz(chunker_spec, worker):
     # Fuzz with random and uniform data of misc. sizes and misc keys.
-    cls, algo, params, env_var = chunker_spec
+    cls, algo, params = chunker_spec
 
     # decompose <ALGO>_PARAMS = (algo, min_exp, max_exp, mask_bits, nc_level)
     params_algo, min_exp, max_exp, mask_bits, nc_level = params

From 2163abfe193e51daad8c91720edb859d2db61e0a Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 13:49:40 +0200
Subject: [PATCH 037/144] buzhash64: keep the NEON kernel, selectable but not
 auto-selected

Restores the NEON kernel removed in 193a26f4e, with BZ_K_AUTO still
resolving to blockwise on aarch64. BORG_BUZHASH64_KERNEL=neon selects it.

Removing it outright over-generalized the measurement. It loses to
blockwise by ~9% on an Apple M3 Pro P-core, but the reason is core width:
the 16 table lookups per block must happen in general registers (NEON has
no gather), so the vector form only ADDS the move to the SIMD side plus a
cross-lane reduce before the loop branch resolves - all it saves is the
8-lane test, three cheap ops per lane, which Apple's very wide scalar
ALUs retire at more than one lane per cycle anyway.

That measurement comes from the widest scalar ARM core in existence.
Neoverse (Graviton, Ampere), Cortex-A7x and friends are 3-4 wide with
comparatively healthy NEON, which is exactly where this should get
competitive - and on this machine's much narrower E-cores the 9% gap
already collapses into measurement noise. Deciding for all of aarch64
from one unrepresentative core was the wrong call; keeping the kernel
costs a dispatch branch and makes flipping the default a one-line change
once someone benchmarks such hardware.

The cross-kernel identity tests now iterate over every kernel the
platform accepts instead of just default-vs-reference, skipping the ones
this build or CPU cannot run. That is what keeps a selectable but
non-default kernel from bit-rotting, and it also covers whichever tier a
given CI machine happens to have.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                              |  6 +-
 src/borg/chunkers/buzhash64.pyx               |  5 +-
 src/borg/chunkers/buzhash64_impl.c            | 99 +++++++++++++++----
 src/borg/chunkers/buzhash64_impl.h            |  8 +-
 src/borg/testsuite/chunkers/buzhash64_test.py | 38 ++++---
 src/borg/testsuite/chunkers/fastcdc_test.py   | 40 +++++---
 .../testsuite/chunkers/phte_chunkers_test.py  | 39 +++++---
 7 files changed, 162 insertions(+), 73 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 28872ef66b..592974bc66 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -189,9 +189,11 @@ New features:
   - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043:
 
     - AVX-512 / AVX2 on x86-64 (Intel / AMD)
-    - fastcdc also NEON on aarch64 (e.g. Apple Silicon); buzhash64 is faster
-      blockwise there, so it has no vector kernel on aarch64
+    - NEON on aarch64 (e.g. Apple Silicon); the default for fastcdc, but not
+      for buzhash64, which is faster blockwise on the cores measured so far
     - blockwise scalar elsewhere
+    - the kernel can be pinned via BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL
+      / BORG_AES_CHUNKER_KERNEL
   - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers
     (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987
   - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64
diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx
index 26c94de41e..cacee641be 100644
--- a/src/borg/chunkers/buzhash64.pyx
+++ b/src/borg/chunkers/buzhash64.pyx
@@ -174,7 +174,10 @@ cdef class ChunkerBuzHash64(ChunkerBase):
 
     @property
     def kernel(self):
-        """Which scan kernel this chunker uses: 'avx512', 'avx2', 'blockwise' or 'scalar'.
+        """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'.
+
+        'neon' exists on aarch64 but is never auto-selected (it loses to
+        'blockwise' there); BORG_BUZHASH64_KERNEL=neon selects it.
 
         With BORG_BUZHASH64_KERNEL set to anything but "auto", this is always the
         requested kernel - creating the chunker fails otherwise.
diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index 4284a6d6d3..1f24d4eedd 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -22,8 +22,8 @@
  * also removes one rotate per byte from the sequential path.
  *
  * Kernel dispatch: AVX-512 or AVX2 on x86-64 (runtime-detected), blockwise
- * scalar everywhere else including aarch64 (which has no vector kernel, see
- * the note where it would go), sequential when forced.
+ * scalar everywhere else, including aarch64 by default - its NEON kernel is
+ * selectable by name but not auto-selected, see the note above it.
  * All kernels return bit-identical results. */
 
 #include <stdlib.h>
@@ -114,29 +114,79 @@ static size_t bz64_scan_blockwise(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
-/* --- aarch64: no vector kernel ------------------------------------------
+/* --- NEON (aarch64, selectable but not the default) ---------------------
  *
- * There was a NEON kernel here doing the 8-lane test in vector registers.
- * It is gone because it lost to the blockwise scalar kernel: measured on an
- * Apple M3 Pro (48 MiB, window 4095), blockwise 2590 MB/s vs NEON 2360 MB/s,
- * the same ~9% gap at every mask size from 17 to 23 bits.
+ * This kernel is NOT auto-selected: on an Apple M3 Pro it loses to the
+ * blockwise scalar one (2590 vs 2360 MB/s, the same ~9% gap at every mask
+ * size from 17 to 23 bits), so bz64_simd_available() returns 0 here and
+ * BZ_K_AUTO resolves to blockwise. BORG_BUZHASH64_KERNEL=neon selects it.
  *
- * The reason is that nothing here needs a vector register in the first
- * place. clang keeps the block's eight prefix XORs in general registers and
- * the scalar test is xor/and/compare per lane, which the wide scalar ALUs
- * retire at more than one lane per cycle; the vector form has to move those
- * eight values across to the SIMD side and then reduce the 8-lane result
- * back to one condition flag (umaxv, ~4 cycles) before the loop branch can
- * resolve. The move-and-reduce round trip costs more than the parallel test
- * saves. fastcdc keeps its NEON kernel because its test is add/and/compare
- * with per-lane shifted masks - enough extra per-lane work to pay for the
- * trip - and there it wins by 2x.
+ * Why it loses there: the 16 table lookups per block have to happen in
+ * general registers (NEON has no gather), so the vector form only ADDS the
+ * move to the SIMD side plus a cross-lane reduce (umaxv) before the loop
+ * branch can resolve - all it saves is the 8-lane test, three cheap ops per
+ * lane, which Apple's very wide scalar ALUs retire at more than one lane per
+ * cycle anyway. fastcdc keeps NEON as its default because its per-lane test
+ * work is larger (add plus per-lane shifted masks), enough to pay for the
+ * trip; there it wins by 2x.
  *
- * So aarch64 falls through to the blockwise scalar kernel below. */
+ * It is kept because that reasoning is about core width, and the measurement
+ * comes from the widest scalar ARM core there is. Neoverse (Graviton,
+ * Ampere), Cortex-A7x and friends are 3-4 wide on the scalar side with
+ * comparatively healthy NEON, which is exactly where this should get
+ * competitive - on this machine's much narrower E-cores the 9% gap already
+ * collapses into measurement noise. If you have such hardware, compare
+ * BORG_BUZHASH64_KERNEL=neon against =blockwise and please report; flipping
+ * the default is then a one-line change here. */
+
+#if defined(__aarch64__)
+#define BZ_KIND "neon"
+
+#include <arm_neon.h>
+
+static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
+                             const uint8_t *pr, const uint8_t *pa,
+                             size_t n, uint64_t *sum_io, uint64_t mask)
+{
+    uint64_t sum = *sum_io;
+    uint64_t s[8];
+    size_t j = 0;
+    uint64x2_t M12 = {BZ_ROTL(mask, 7), BZ_ROTL(mask, 6)};
+    uint64x2_t M34 = {BZ_ROTL(mask, 5), BZ_ROTL(mask, 4)};
+    uint64x2_t M56 = {BZ_ROTL(mask, 3), BZ_ROTL(mask, 2)};
+    uint64x2_t M78 = {BZ_ROTL(mask, 1), mask};
+
+    while (j + 8 <= n && (sum & mask) != 0) {
+        bz64_block_prefix(T, Trot, pr + j, pa + j, s);
+        uint64_t c = BZ_ROTL(sum, 8);
+        uint64x2_t Cv = vdupq_n_u64(c);
+        uint64x2_t z12 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[0], s[1]}), M12));
+        uint64x2_t z34 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[2], s[3]}), M34));
+        uint64x2_t z56 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[4], s[5]}), M56));
+        uint64x2_t z78 = vceqzq_u64(vandq_u64(veorq_u64(Cv, (uint64x2_t){s[6], s[7]}), M78));
+        uint64x2_t any = vorrq_u64(vorrq_u64(z12, z34), vorrq_u64(z56, z78));
+        if (vmaxvq_u32(vreinterpretq_u32_u64(any))) {
+            size_t r = bz64_scan_seq(T, Trot, pr + j, pa + j, 8, &sum, mask); /* exact re-scan */
+            *sum_io = sum;
+            return j + r;
+        }
+        sum = c ^ s[7];
+        j += 8;
+    }
+    if (j < n)
+        j += bz64_scan_seq(T, Trot, pr + j, pa + j, n - j, &sum, mask);
+    *sum_io = sum;
+    return j;
+}
+
+static int bz64_simd_available(void)
+{
+    return 0; /* see above: selectable by name, but never auto-selected */
+}
 
 /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */
 
-#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
 #define BZ_KIND "avx2"
 
 #include <immintrin.h>
@@ -328,7 +378,9 @@ static int bz64_simd_available(void)
 
 const char *bz64_kernel_names(void)
 {
-#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+#if defined(__aarch64__)
+    return "auto, neon, blockwise, scalar";
+#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
     return "auto, avx512, avx2, blockwise, scalar";
 #else
     return "auto, blockwise, scalar";
@@ -349,7 +401,12 @@ int bz64_kernel_select(const char *name, int *out_id)
         *out_id = BZ_K_BLOCKWISE;
         return BZ_KSEL_OK;
     }
-#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
+#if defined(__aarch64__)
+    if (strcmp(name, "neon") == 0) {
+        *out_id = BZ_K_VECTOR; /* baseline on aarch64, always runnable */
+        return BZ_KSEL_OK;
+    }
+#elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
     if (strcmp(name, "avx2") == 0) {
         if (!__builtin_cpu_supports("avx2"))
             return BZ_KSEL_NOCPU;
diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h
index f6dbfdd900..c75496ce35 100644
--- a/src/borg/chunkers/buzhash64_impl.h
+++ b/src/borg/chunkers/buzhash64_impl.h
@@ -27,7 +27,7 @@
 #define BZ_K_AUTO 0      /* best kernel this CPU can run */
 #define BZ_K_SCALAR 1    /* sequential reference loop */
 #define BZ_K_BLOCKWISE 2 /* portable 8-lane C */
-#define BZ_K_VECTOR 3    /* avx2 (there is no aarch64 vector kernel) */
+#define BZ_K_VECTOR 3    /* neon / avx2 (neon is not auto-selected) */
 #define BZ_K_VECTOR512 4 /* avx512 */
 
 /* Results of bz64_kernel_select(). */
@@ -46,9 +46,9 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                  const uint8_t *p_rem, const uint8_t *p_add,
                  size_t n, uint64_t *sum, uint64_t mask, int kernel);
 
-/* Name of the kernel <kernel> selects: "avx512", "avx2", "blockwise" or
- * "scalar" (there is no vector kernel on aarch64, see buzhash64_impl.c).
- * For BZ_K_AUTO, the auto-selected one. */
+/* Name of the kernel <kernel> selects: "neon", "avx512", "avx2", "blockwise"
+ * or "scalar"; for BZ_K_AUTO, the auto-selected one. Note "neon" is never
+ * auto-selected, see buzhash64_impl.c. */
 const char *bz64_kernel_name(int kernel);
 
 #endif
diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py
index eec877000b..0848e8f9be 100644
--- a/src/borg/testsuite/chunkers/buzhash64_test.py
+++ b/src/borg/testsuite/chunkers/buzhash64_test.py
@@ -142,24 +142,32 @@ def rnd_key():
             assert reconstructed == data
 
 
-def test_buzhash64_kernels_identical():
-    # the SIMD scan kernel (avx512/avx2/blockwise, auto-selected) and the plain sequential
-    # loop must produce identical cut points.
+ALL_BUZHASH64_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar")
+
+
+def test_buzhash64_kernels_identical(monkeypatch):
+    # Every scan kernel this platform accepts must produce identical cut points;
+    # see the fastcdc counterpart. On aarch64 this is what keeps the NEON kernel
+    # verified - it is selectable but never auto-selected.
     data = os.urandom(4 * 1024 * 1024)
     key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
 
     def sizes(chunker):
         return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))]
 
-    default = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
-    sizes_default = sizes(default)
-    os.environ["BORG_BUZHASH64_KERNEL"] = "scalar"
-    try:
-        forced = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
-        assert forced.kernel == "scalar"
-        sizes_scalar = sizes(forced)
-    finally:
-        del os.environ["BORG_BUZHASH64_KERNEL"]
-    assert sizes_default == sizes_scalar
-    # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("avx512", "avx2", "blockwise", "scalar")
+    monkeypatch.delenv("BORG_BUZHASH64_KERNEL", raising=False)
+    reference = sizes(ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2))
+
+    tested = []
+    for name in ALL_BUZHASH64_KERNELS:
+        monkeypatch.setenv("BORG_BUZHASH64_KERNEL", name)
+        try:
+            chunker = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
+        except ValueError:
+            continue  # not available on this build/CPU
+        if name != "auto":
+            assert chunker.kernel == name
+        assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one"
+        tested.append(name)
+
+    assert "blockwise" in tested and "scalar" in tested
diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py
index 8d860c2ab5..9e19870ad9 100644
--- a/src/borg/testsuite/chunkers/fastcdc_test.py
+++ b/src/borg/testsuite/chunkers/fastcdc_test.py
@@ -178,27 +178,37 @@ def rnd_key():
             assert b"".join(parts) == data
 
 
-def test_fastcdc_kernels_identical():
-    # the SIMD scan kernel (neon/avx512/avx2/blockwise, auto-selected) and the
-    # plain sequential Gear loop must produce identical cut points.
+ALL_FASTCDC_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar")
+
+
+def test_fastcdc_kernels_identical(monkeypatch):
+    # Every scan kernel this platform accepts must produce identical cut points.
+    # Kernels this build/CPU cannot run raise and are skipped, so the same test
+    # covers whatever tier the machine happens to have - including kernels that
+    # exist but are not auto-selected, which nothing else would exercise.
     data = os.urandom(4 * 1024 * 1024)
     key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
 
     def sizes(chunker):
         return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))]
 
-    default = ChunkerFastCDC(key0, 10, 16, 14, 2)
-    sizes_default = sizes(default)
-    os.environ["BORG_FASTCDC_KERNEL"] = "scalar"
-    try:
-        forced = ChunkerFastCDC(key0, 10, 16, 14, 2)
-        assert forced.kernel == "scalar"
-        sizes_scalar = sizes(forced)
-    finally:
-        del os.environ["BORG_FASTCDC_KERNEL"]
-    assert sizes_default == sizes_scalar
-    # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("neon", "avx512", "avx2", "blockwise", "scalar")
+    monkeypatch.delenv("BORG_FASTCDC_KERNEL", raising=False)
+    reference = sizes(ChunkerFastCDC(key0, 10, 16, 14, 2))
+
+    tested = []
+    for name in ALL_FASTCDC_KERNELS:
+        monkeypatch.setenv("BORG_FASTCDC_KERNEL", name)
+        try:
+            chunker = ChunkerFastCDC(key0, 10, 16, 14, 2)
+        except ValueError:
+            continue  # not available on this build/CPU
+        if name != "auto":
+            assert chunker.kernel == name
+        assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one"
+        tested.append(name)
+
+    # the portable kernels exist everywhere; if they were skipped, the loop is broken
+    assert "blockwise" in tested and "scalar" in tested
 
 
 @pytest.mark.parametrize("envvar", ["BORG_FASTCDC_KERNEL", "BORG_BUZHASH64_KERNEL", "BORG_AES_CHUNKER_KERNEL"])
diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py
index 50b54220f5..19b89ffac7 100644
--- a/src/borg/testsuite/chunkers/phte_chunkers_test.py
+++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py
@@ -44,27 +44,36 @@ def chunker_spec(request):
     return request.param
 
 
-def test_kernels_identical(chunker_spec):
-    # the OpenSSL EVP batch path and the AES hardware instruction path (if available
-    # on this platform) must produce identical cut points.
+ALL_AES_KERNELS = ("auto", "vaes", "aes-ni", "aes-arm64", "evp")
+
+
+def test_kernels_identical(chunker_spec, monkeypatch):
+    # Every scan path this platform accepts - OpenSSL EVP, the 128-bit AES
+    # instructions, VAES - must produce identical cut points. Paths this
+    # build/CPU cannot run raise and are skipped.
     cls, algo, params = chunker_spec
     data = os.urandom(4 * 1024 * 1024)
 
     def sizes(chunker):
         return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))]
 
-    default = cls(key0, 10, 16, 14, 2)
-    sizes_default = sizes(default)
-    os.environ[KERNEL_ENV] = "evp"
-    try:
-        forced = cls(key0, 10, 16, 14, 2)
-        assert forced.kernel == "evp"
-        sizes_evp = sizes(forced)
-    finally:
-        del os.environ[KERNEL_ENV]
-    assert sizes_default == sizes_evp
-    # whatever kernel was selected by default, it must be a known one
-    assert default.kernel in ("aes-arm64", "vaes", "aes-ni", "evp")
+    monkeypatch.delenv(KERNEL_ENV, raising=False)
+    reference = sizes(cls(key0, 10, 16, 14, 2))
+
+    tested = []
+    for name in ALL_AES_KERNELS:
+        monkeypatch.setenv(KERNEL_ENV, name)
+        try:
+            chunker = cls(key0, 10, 16, 14, 2)
+        except ValueError:
+            continue  # not available on this build/CPU
+        if name != "auto":
+            assert chunker.kernel == name
+        assert sizes(chunker) == reference, f"path {name} disagrees with the default one"
+        tested.append(name)
+
+    # the portable OpenSSL path exists everywhere
+    assert "evp" in tested
 
 
 def test_chunksize_distribution(chunker_spec):

From f00a8cee5512034d2ed8524dbffe4136ade65560 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 6 Aug 2026 17:31:42 +0530
Subject: [PATCH 038/144] check: fix partial-run startup log line, soften
 --max-age cadence wording

---
 src/borg/archiver/check_cmd.py | 3 ++-
 src/borg/repository.py         | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 36e3ceda1b..6e983a9876 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -148,7 +148,8 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         interrupted. A partial check verifies the least-recently-checked packs first,
         so repeated runs cover the whole repository. Add ``--max-age`` to also skip
         packs whose result is still younger than the given age: once every pack has a
-        recent result, further runs re-check each pack about once per ``--max-age``.
+        recent result, further runs re-check each pack at most once per ``--max-age``,
+        and no faster than the per-run budget allows.
         Assuming a complete check would take 7 hours, running a daily check with
         ``--max-duration=3600 --max-age=1w`` (1 hour) results in one full repository
         verification per week. Partial repository checks run neither archive checks
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 6c3683b640..b874f8acff 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -871,6 +871,10 @@ def store_list(namespace):
             logger.info("Starting from beginning.")
         elif max_age:
             logger.info(f"{len(tracker)} pack check results on record, reusing those younger than --max-age.")
+        elif partial:
+            logger.info(
+                f"{len(tracker)} pack check results on record, verifying the least-recently-checked packs first."
+            )
         else:
             logger.info(f"{len(tracker)} pack check results on record, verifying every pack.")
         t_start = time.monotonic()

From eaef6ac879a58b5db251d10c5ff62478bb386634 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 29 Jul 2026 18:08:30 +0530
Subject: [PATCH 039/144] check: handle soft interrupt (Ctrl-C) at safe
 boundaries, always run finish() (#7893)

---
 src/borg/archive.py                           | 30 +++++++++--
 src/borg/archiver/check_cmd.py                |  6 ++-
 src/borg/repository.py                        |  4 ++
 src/borg/testsuite/archiver/check_cmd_test.py | 54 +++++++++++++++++++
 4 files changed, 87 insertions(+), 7 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 271118ac86..c4516f0644 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -32,7 +32,7 @@
 from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError
 from .helpers import HardLinkManager
 from .helpers import ChunkIteratorFileWrapper, open_item
-from .helpers import Error, IntegrityError, set_ec
+from .helpers import Error, IntegrityError, set_ec, sig_int
 from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns
 from .helpers import parse_timestamp, archive_ts_now, CompressionSpec
 from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize
@@ -1929,12 +1929,25 @@ def check(
                 rebuild_manifest = True
         if rebuild_manifest:
             self.manifest = self.rebuild_manifest()
-        if find_lost_archives:
+        # Skip the remaining scans on Ctrl-C, but still run finish() below.
+        if find_lost_archives and not sig_int:
             self.rebuild_archives_directory()
-        self.rebuild_archives(
-            match=match, first=first, last=last, sort_by=sort_by, older=older, oldest=oldest, newer=newer, newest=newest
-        )
+        if not sig_int:
+            self.rebuild_archives(
+                match=match,
+                first=first,
+                last=last,
+                sort_by=sort_by,
+                older=older,
+                oldest=oldest,
+                newer=newer,
+                newest=newest,
+            )
+        # finish() drops the chunk index and writes the manifest; run it even on Ctrl-C so --repair
+        # leaves a valid index (#9850).
         self.finish()
+        if sig_int:
+            raise Error("Got Ctrl-C / SIGINT.")
         if self.error_found:
             logger.error("Archive consistency check complete, problems found.")
         else:
@@ -1987,6 +2000,8 @@ def verify_data(self):
             total=chunks_count, msg="Verifying data %6.2f%%", step=0.01, msgid="check.verify_data"
         )
         for chunk_id, _ in self.chunks.iteritems():
+            if sig_int:  # stop at a chunk boundary
+                break
             pi.show()
             try:
                 encrypted_data = self.repository.get(chunk_id)
@@ -2083,6 +2098,8 @@ def valid_archive(obj):
             msgid="check.rebuild_archives_directory",
         )
         for chunk_id, _ in self.chunks.iteritems():
+            if sig_int:  # stop at a chunk boundary
+                break
             pi.show()
             cdata = self.repository.get(chunk_id, read_data=False)  # only get metadata
             try:
@@ -2326,6 +2343,9 @@ def valid_item(obj):
         # badly damaged repo does not throw away everything it already found.
         try:
             for i, info in enumerate(archive_infos):
+                if sig_int:
+                    # Break only between archives: --repair rewrites each archive as a whole below.
+                    break
                 pi.show(i)
                 archive_id, archive_id_hex = info.id, bin_to_hex(info.id)
                 try:
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 682f3a9279..b2037abff2 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -3,8 +3,8 @@
 from ._common import with_repository, Highlander
 from ..archive import ArchiveChecker
 from ..constants import *  # NOQA
-from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError
-from ..helpers import yes, ArchiveFormatter
+from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, Error, IntegrityError
+from ..helpers import yes, ArchiveFormatter, sig_int
 from ..helpers.argparsing import ArgumentParser
 
 from ..logger import create_logger
@@ -66,6 +66,8 @@ def do_check(self, args, repository):
         if not args.archives_only:
             if not repository.check(repair=args.repair, max_duration=args.max_duration):
                 set_ec(EXIT_WARNING)
+            if sig_int:  # repository check interrupted; skip the archive check
+                raise Error("Got Ctrl-C / SIGINT.")
         if not args.repo_only and not archive_checker.check(
             repository,
             verify_data=args.verify_data,
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 3bb3e3ed4a..900eede627 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1019,6 +1019,10 @@ def store_list(namespace):
             pack_infos = store_list("packs")
             pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs")
             for info in pack_infos:
+                if sig_int:  # save progress so a later check resumes, then stop
+                    logger.info(f"Interrupted repository check, {len(tracker)} packs checked so far.")
+                    tracker.save()
+                    break
                 self._lock_refresh()
                 pack_pi.show(increase=1)  # advance for skipped packs too, so the bar tracks packs/, not work done
                 pack_id = hex_to_bin(info.name)
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 64c3fc990f..767b1c4ebb 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -75,6 +75,60 @@ def test_check_usage(archivers, request):
     assert "archive2" in output
 
 
+def test_check_soft_interrupt(archivers, request):
+    """A Ctrl-C during a read-only check stops at a safe boundary and raises 'Got Ctrl-C' (#7893).
+    It changes nothing, so a normal check still passes afterwards."""
+    from ...archive import ArchiveChecker
+    from ...helpers import sig_int, Error
+
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+
+    try:
+        with Repository(archiver.repository_path, exclusive=True) as repository:
+            # repository check: stops at a pack boundary, still reports no errors.
+            sig_int._sig_int_triggered = True
+            assert repository.check() is True
+        with Repository(archiver.repository_path, exclusive=True) as repository:
+            # archive check: runs finish(), then raises.
+            sig_int._sig_int_triggered = True
+            with pytest.raises(Error, match="Got Ctrl-C"):
+                ArchiveChecker().check(repository, verify_data=True, sort_by="ts", format="{archive} {time} {id}")
+    finally:
+        sig_int._sig_int_triggered = False  # reset the global flag for the following tests
+
+    cmd(archiver, "check", exit_code=0)
+
+
+def test_check_repair_soft_interrupt(archivers, request, monkeypatch):
+    """A Ctrl-C after the first archive of a --repair archive check stops at the archive boundary, runs
+    finish() (dropping the chunk index, writing the manifest), then raises. A later check confirms the
+    repository is still consistent."""
+    from ...archive import ArchiveChecker
+    from ...manifest import Archives
+    from ...helpers import sig_int, Error
+
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)  # two archives
+
+    orig_create = Archives.create
+
+    def create_then_interrupt(self, *args, **kwargs):
+        orig_create(self, *args, **kwargs)
+        sig_int._sig_int_triggered = True  # one Ctrl-C after the first archive was rebuilt
+
+    monkeypatch.setattr(Archives, "create", create_then_interrupt)
+    try:
+        with Repository(archiver.repository_path, exclusive=True) as repository:
+            with pytest.raises(Error, match="Got Ctrl-C"):
+                ArchiveChecker().check(repository, repair=True, sort_by="ts", format="{archive} {time} {id}")
+    finally:
+        sig_int._sig_int_triggered = False  # reset the global flag for the following tests
+
+    # a normal check does not rebuild archives, so the patched Archives.create never fires here
+    cmd(archiver, "check", exit_code=0)
+
+
 def test_date_matching(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)

From 566f643aad379bc74bf13bed6c368182398d3d8c Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 5 Aug 2026 18:08:08 +0530
Subject: [PATCH 040/144] check: report honest status on Ctrl-C interrupt

verify_data() now logs how many chunks it actually verified and an
interrupted variant of its summary; Repository.check() and
rebuild_archives_directory() likewise report interruption instead of
success/completion when stopped by SIGINT.

Document the SIGINT contract in the check epilog and rework the
soft-interrupt tests to interrupt mid-run and assert the resulting
state (persisted pack-check progress, both archives surviving a
--repair interrupt, a second --repair finishing the job).
---
 src/borg/archive.py                           | 41 +++++---
 src/borg/archiver/check_cmd.py                |  8 ++
 src/borg/repository.py                        |  7 +-
 src/borg/testsuite/archiver/check_cmd_test.py | 94 +++++++++++++------
 4 files changed, 105 insertions(+), 45 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index c4516f0644..7e297274b4 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -1929,10 +1929,10 @@ def check(
                 rebuild_manifest = True
         if rebuild_manifest:
             self.manifest = self.rebuild_manifest()
-        # Skip the remaining scans on Ctrl-C, but still run finish() below.
-        if find_lost_archives and not sig_int:
-            self.rebuild_archives_directory()
+        # On Ctrl-C, skip the remaining scans.
         if not sig_int:
+            if find_lost_archives:
+                self.rebuild_archives_directory()
             self.rebuild_archives(
                 match=match,
                 first=first,
@@ -1943,8 +1943,7 @@ def check(
                 newer=newer,
                 newest=newest,
             )
-        # finish() drops the chunk index and writes the manifest; run it even on Ctrl-C so --repair
-        # leaves a valid index (#9850).
+        # finish() writes the manifest and a consistent chunk index; run it on Ctrl-C too (#9850).
         self.finish()
         if sig_int:
             raise Error("Got Ctrl-C / SIGINT.")
@@ -1995,14 +1994,16 @@ def verify_data(self):
         logger.info("Starting cryptographic data integrity verification...")
         chunks_count = len(self.chunks)
         errors = 0
+        verified = 0  # chunks actually verified
         defect_chunks = []
         pi = ProgressIndicatorPercent(
             total=chunks_count, msg="Verifying data %6.2f%%", step=0.01, msgid="check.verify_data"
         )
         for chunk_id, _ in self.chunks.iteritems():
-            if sig_int:  # stop at a chunk boundary
+            if sig_int:
                 break
             pi.show()
+            verified += 1
             try:
                 encrypted_data = self.repository.get(chunk_id)
             except (Repository.ObjectNotFound, IntegrityErrorBase) as err:
@@ -2058,11 +2059,20 @@ def verify_data(self):
                 for defect_chunk in defect_chunks:
                     logger.debug("chunk %s is defect.", bin_to_hex(defect_chunk))
         log = logger.error if errors else logger.info
-        log(
-            "Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.",
-            chunks_count,
-            errors,
-        )
+        if sig_int:
+            log(
+                "Interrupted cryptographic data integrity verification, "
+                "verified %d of %d chunks with %d integrity errors.",
+                verified,
+                chunks_count,
+                errors,
+            )
+        else:
+            log(
+                "Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.",
+                verified,
+                errors,
+            )
 
     def rebuild_manifest(self):
         """Rebuild the manifest object."""
@@ -2098,7 +2108,7 @@ def valid_archive(obj):
             msgid="check.rebuild_archives_directory",
         )
         for chunk_id, _ in self.chunks.iteritems():
-            if sig_int:  # stop at a chunk boundary
+            if sig_int:
                 break
             pi.show()
             cdata = self.repository.get(chunk_id, read_data=False)  # only get metadata
@@ -2145,7 +2155,10 @@ def valid_archive(obj):
                         logger.warning(f"Would create archives directory entry for {name} {archive_id_hex}.")
 
         pi.finish()
-        logger.info("Rebuilding missing archives directory entries completed.")
+        if sig_int:
+            logger.info("Rebuilding missing archives directory entries interrupted.")
+        else:
+            logger.info("Rebuilding missing archives directory entries completed.")
 
     def rebuild_archives(
         self, first=0, last=0, sort_by="", match=None, older=None, newer=None, oldest=None, newest=None
@@ -2344,7 +2357,7 @@ def valid_item(obj):
         try:
             for i, info in enumerate(archive_infos):
                 if sig_int:
-                    # Break only between archives: --repair rewrites each archive as a whole below.
+                    # Break only between archives, as --repair rewrites each archive as a whole.
                     break
                 pi.show(i)
                 archive_id, archive_id_hex = info.id, bin_to_hex(info.id)
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index b2037abff2..5800b3175e 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -162,6 +162,14 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         formatted by giving a custom format using ``--format`` (see the ``borg repo-list``
         description for more details about the format string).
 
+        If the ``borg check`` process receives a SIGINT signal (Ctrl-C), it stops at the
+        next safe boundary (a pack boundary during the repository check, an archive boundary
+        during the archive check), leaving the repository and its chunk index in a consistent
+        state. A partial repository check (``--max-duration``) saves its progress so a later
+        partial check resumes where it stopped; a full check restarts from the beginning.
+        With ``--repair``, an interrupted archive check may leave some archives already
+        repaired and others not yet processed, so run ``borg check --repair`` again to finish.
+
         About repair mode
         +++++++++++++++++
 
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 900eede627..02667de446 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1019,7 +1019,7 @@ def store_list(namespace):
             pack_infos = store_list("packs")
             pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs")
             for info in pack_infos:
-                if sig_int:  # save progress so a later check resumes, then stop
+                if sig_int:  # on Ctrl-C, persist checked packs, then stop
                     logger.info(f"Interrupted repository check, {len(tracker)} packs checked so far.")
                     tracker.save()
                     break
@@ -1059,7 +1059,10 @@ def store_list(namespace):
             f"Checked {index_files} index files ({index_errors} errors) and {pack_files} packs ({pack_errors} errors)."
         )
         if objs_errors == 0:
-            logger.info(f"Finished {mode} repository check, no problems found.")
+            if sig_int:
+                logger.info(f"Interrupted {mode} repository check, no problems found so far.")
+            else:
+                logger.info(f"Finished {mode} repository check, no problems found.")
         elif repair:
             logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).")
         else:
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 767b1c4ebb..41967cea04 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -6,11 +6,11 @@
 
 import pytest
 
-from ...archive import ChunkBuffer, ArchiveChecker
+from ...archive import ArchiveChecker, ChunkBuffer
 from ...constants import *  # NOQA
-from ...helpers import bin_to_hex, msgpack, CommandError, IntegrityError
-from ...manifest import Manifest
-from ...repository import Repository
+from ...helpers import bin_to_hex, msgpack, CommandError, Error, IntegrityError, sig_int
+from ...manifest import Archives, Manifest
+from ...repository import PackTracker, Repository
 from ..repository_test import fchunk, corrupt_chunk_on_disk
 from . import (
     cmd,
@@ -75,39 +75,67 @@ def test_check_usage(archivers, request):
     assert "archive2" in output
 
 
-def test_check_soft_interrupt(archivers, request):
-    """A Ctrl-C during a read-only check stops at a safe boundary and raises 'Got Ctrl-C' (#7893).
-    It changes nothing, so a normal check still passes afterwards."""
-    from ...archive import ArchiveChecker
-    from ...helpers import sig_int, Error
-
+def test_check_soft_interrupt(archivers, request, monkeypatch):
+    """A mid-run Ctrl-C stops both check phases at a safe boundary (#7893): the repository check persists
+    its progress for a later partial check to resume, the archive check runs finish() and then raises.
+    The check is read-only, so a normal check still passes afterwards."""
     archiver = request.getfixturevalue(archivers)
-    check_cmd_setup(archiver)
-
-    try:
-        with Repository(archiver.repository_path, exclusive=True) as repository:
-            # repository check: stops at a pack boundary, still reports no errors.
-            sig_int._sig_int_triggered = True
-            assert repository.check() is True
-        with Repository(archiver.repository_path, exclusive=True) as repository:
-            # archive check: runs finish(), then raises.
-            sig_int._sig_int_triggered = True
+    check_cmd_setup(archiver)  # produces many packs
+
+    # repository check: interrupt after a few packs.
+    with Repository(archiver.repository_path, exclusive=True) as repository:
+        orig_hash = repository.store.hash
+        hash_calls = 0
+
+        def hash_then_interrupt(key):
+            nonlocal hash_calls
+            hash_calls += 1
+            result = orig_hash(key)
+            if hash_calls == 5:  # trip mid-run, after several packs
+                sig_int._sig_int_triggered = True
+            return result
+
+        monkeypatch.setattr(repository.store, "hash", hash_then_interrupt)
+        try:
+            assert repository.check() is True  # interrupted, no errors found
+        finally:
+            sig_int._sig_int_triggered = False
+        assert len(PackTracker.load(repository.store)) > 0  # the verified packs were persisted
+
+    # a partial check resumes the saved cycle.
+    output = cmd(archiver, "check", "-v", "--repository-only", "--max-duration=600", exit_code=0)
+    assert "Continuing check cycle" in output
+
+    # archive check: interrupt verify_data after 3 chunks.
+    with Repository(archiver.repository_path, exclusive=True) as repository:
+        orig_get = repository.get
+        get_calls = 0
+        interrupted_after = None
+
+        def get_then_interrupt(*args, **kwargs):
+            nonlocal get_calls, interrupted_after
+            get_calls += 1
+            if get_calls == 3:  # trip mid-loop, after 3 chunks
+                sig_int._sig_int_triggered = True
+                interrupted_after = get_calls
+            return orig_get(*args, **kwargs)
+
+        monkeypatch.setattr(repository, "get", get_then_interrupt)
+        try:
             with pytest.raises(Error, match="Got Ctrl-C"):
                 ArchiveChecker().check(repository, verify_data=True, sort_by="ts", format="{archive} {time} {id}")
-    finally:
-        sig_int._sig_int_triggered = False  # reset the global flag for the following tests
+        finally:
+            sig_int._sig_int_triggered = False
+        assert interrupted_after == 3  # the loop stopped mid-run, after verifying 3 chunks
 
+    # nothing changed, so a normal check passes.
     cmd(archiver, "check", exit_code=0)
 
 
 def test_check_repair_soft_interrupt(archivers, request, monkeypatch):
     """A Ctrl-C after the first archive of a --repair archive check stops at the archive boundary, runs
-    finish() (dropping the chunk index, writing the manifest), then raises. A later check confirms the
-    repository is still consistent."""
-    from ...archive import ArchiveChecker
-    from ...manifest import Archives
-    from ...helpers import sig_int, Error
-
+    finish() (dropping the chunk index, writing the manifest), then raises. No archive is lost, and a
+    second --repair finishes the job so a following check reports the repository consistent."""
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)  # two archives
 
@@ -124,8 +152,16 @@ def create_then_interrupt(self, *args, **kwargs):
                 ArchiveChecker().check(repository, repair=True, sort_by="ts", format="{archive} {time} {id}")
     finally:
         sig_int._sig_int_triggered = False  # reset the global flag for the following tests
+    # restore the real method; monkeypatch.undo() would also drop the autouse env (BORG_TESTONLY_WEAKEN_KDF).
+    monkeypatch.setattr(Archives, "create", orig_create)
+
+    # both archives survive the interrupt between archives.
+    output = cmd(archiver, "repo-list", exit_code=0)
+    assert "archive1" in output
+    assert "archive2" in output
 
-    # a normal check does not rebuild archives, so the patched Archives.create never fires here
+    # a second --repair finishes the job; a plain check then finds no problems.
+    cmd(archiver, "check", "--repair", exit_code=0)
     cmd(archiver, "check", exit_code=0)
 
 

From e848f1f98ea80b8cdae9e6e54e501b7aa3e96178 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 6 Aug 2026 04:43:57 +0530
Subject: [PATCH 041/144] check: honest interrupt summary, precise SIGINT docs,
 align tests

- archive check logs an interrupted summary (reflecting whether problems
  were found) before raising, instead of dropping the summary line.
- epilog: scope the not-yet-interruptible note to the archive check's
  chunk-index rebuild and key recovery, not the whole --repair run.
- repository.check(): note what the return value means on interrupt.
- soft-interrupt tests trip after the first pack (packs namespace only)
  and assert persisted state, matching test_compact_soft_interrupt.
---
 src/borg/archive.py                           |  4 ++++
 src/borg/archiver/check_cmd.py                |  5 +++++
 src/borg/repository.py                        |  1 +
 src/borg/testsuite/archiver/check_cmd_test.py | 22 +++++++++----------
 4 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 7e297274b4..337b297394 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -1946,6 +1946,10 @@ def check(
         # finish() writes the manifest and a consistent chunk index; run it on Ctrl-C too (#9850).
         self.finish()
         if sig_int:
+            if self.error_found:
+                logger.error("Archive consistency check interrupted, problems found so far.")
+            else:
+                logger.info("Archive consistency check interrupted, no problems found so far.")
             raise Error("Got Ctrl-C / SIGINT.")
         if self.error_found:
             logger.error("Archive consistency check complete, problems found.")
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 5800b3175e..6a3092e3f2 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -170,6 +170,11 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         With ``--repair``, an interrupted archive check may leave some archives already
         repaired and others not yet processed, so run ``borg check --repair`` again to finish.
 
+        During a ``--repair`` run, the archive check first rebuilds the chunk index from the
+        packs, and, if the key must be recovered, scans chunks for it. These phases do not yet
+        respond to SIGINT, so on a large repository a Ctrl-C during them may appear to have no
+        effect until they finish.
+
         About repair mode
         +++++++++++++++++
 
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 02667de446..ec88a65f3e 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1067,6 +1067,7 @@ def store_list(namespace):
             logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).")
         else:
             logger.error(f"Finished {mode} repository check, errors found.")
+        # True means the checked objects were clean; on Ctrl-C that covers only the packs seen so far.
         return objs_errors == 0 or repair
 
     def list(self, limit=None, marker=None):
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 41967cea04..5d5c21cd4a 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -77,30 +77,30 @@ def test_check_usage(archivers, request):
 
 def test_check_soft_interrupt(archivers, request, monkeypatch):
     """A mid-run Ctrl-C stops both check phases at a safe boundary (#7893): the repository check persists
-    its progress for a later partial check to resume, the archive check runs finish() and then raises.
-    The check is read-only, so a normal check still passes afterwards."""
+    its checked packs for a later partial check to resume, and the archive check runs finish() and then
+    raises. The check is read-only, so a normal check still passes afterwards."""
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)  # produces many packs
 
-    # repository check: interrupt after a few packs.
+    # repository check: interrupt after the first pack.
     with Repository(archiver.repository_path, exclusive=True) as repository:
         orig_hash = repository.store.hash
-        hash_calls = 0
+        pack_checks = []
 
         def hash_then_interrupt(key):
-            nonlocal hash_calls
-            hash_calls += 1
             result = orig_hash(key)
-            if hash_calls == 5:  # trip mid-run, after several packs
-                sig_int._sig_int_triggered = True
+            if key.startswith("packs/"):  # count pack checks, not the index files hashed first
+                pack_checks.append(key)
+                if len(pack_checks) == 1:  # one Ctrl-C after the first pack is checked
+                    sig_int._sig_int_triggered = True
             return result
 
         monkeypatch.setattr(repository.store, "hash", hash_then_interrupt)
         try:
-            assert repository.check() is True  # interrupted, no errors found
+            repository.check()
         finally:
             sig_int._sig_int_triggered = False
-        assert len(PackTracker.load(repository.store)) > 0  # the verified packs were persisted
+        assert len(PackTracker.load(repository.store)) == 1  # the pack checked before the break persisted
 
     # a partial check resumes the saved cycle.
     output = cmd(archiver, "check", "-v", "--repository-only", "--max-duration=600", exit_code=0)
@@ -126,7 +126,7 @@ def get_then_interrupt(*args, **kwargs):
                 ArchiveChecker().check(repository, verify_data=True, sort_by="ts", format="{archive} {time} {id}")
         finally:
             sig_int._sig_int_triggered = False
-        assert interrupted_after == 3  # the loop stopped mid-run, after verifying 3 chunks
+        assert interrupted_after == 3  # the loop stopped after verifying 3 chunks
 
     # nothing changed, so a normal check passes.
     cmd(archiver, "check", exit_code=0)

From f71c3def0df2fea9226289221347b7164b33bb6e Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 14:19:54 +0200
Subject: [PATCH 042/144] build: bundle borg.chunkers.kernel_env into the
 binary

The binary build broke with

  File "src/borg/chunkers/buzhash64.pyx", line 26, in init borg.chunkers.buzhash64
    from .kernel_env import kernel_error, requested_kernel
  ModuleNotFoundError: No module named 'borg.chunkers.kernel_env'

kernel_env is imported only from .pyx files, and the chunkers are Cython
extensions, so PyInstaller's static analysis of the Python sources cannot
see the import at all and never collects the module. The spec already
lists borg.chunkers.base and .phte_chunker as hiddenimports for the same
reason (they are cimported at C level); this adds kernel_env and widens
the comment, since the rule is "anything a compiled chunker pulls in has
to be listed here", not just the cimported base classes.

Verified both ways with a local pyinstaller build: with the entry the
binary runs "borg benchmark cpu" through all chunkers, and rebuilding
with the entry removed reproduces the traceback above exactly.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 scripts/borg.exe.spec | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scripts/borg.exe.spec b/scripts/borg.exe.spec
index a705db35b6..e1155cad69 100644
--- a/scripts/borg.exe.spec
+++ b/scripts/borg.exe.spec
@@ -14,12 +14,16 @@ if is_win32:
 else:
     hiddenimports = ['borg.platform.posix', 'borghash', 'rich._unicode_data.unicode17-0-0']
 
-# The chunkers cimport their shared base classes (ChunkerBase, and ChunkerPHTE
-# for the *-aes chunkers), so they import those modules at C level when
-# initializing - PyInstaller can not see this by static analysis of the Python
-# sources.
+# Anything a compiled chunker module pulls in has to be listed here: the
+# chunkers are Cython extensions, so PyInstaller's static analysis of the
+# Python sources cannot see their imports at all. That covers both the base
+# classes they cimport (ChunkerBase, and ChunkerPHTE for the *-aes chunkers),
+# which are imported at C level during module init, and plain Python modules
+# imported from a .pyx, such as kernel_env - it is imported ONLY from .pyx
+# files, so nothing else would drag it in.
 hiddenimports.append('borg.chunkers.base')
 hiddenimports.append('borg.chunkers.phte_chunker')
+hiddenimports.append('borg.chunkers.kernel_env')
 
 block_cipher = None
 

From 5257426e533186e66f114aaece137381eecaed50 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 6 Aug 2026 19:43:24 +0530
Subject: [PATCH 043/144] check: honest interrupt summary in all branches,
 precise boundaries, cover skip

Repository.check() only reported "Interrupted" when no problems were found;
the error branches still logged "Finished" on Ctrl-C. Thread the interrupted
tense and "so far" through all three branches so an interrupted check that
also found errors is reported honestly.

Split the archive-check sig_int guard so a Ctrl-C during rebuild_archives_directory
skips rebuild_archives instead of entering it and listing the archives before
breaking at the first one.

Reword the epilog to name the actual stop boundaries: after the current pack for
the repository check, after the current chunk for --verify-data and
--find-lost-archives, and between whole archives for a --repair archive check.

Assert get_calls == 3 instead of the tautological interrupted_after == 3, and add
a test that drives borg check through do_check so an interrupted repository check
skips the archive check.
---
 src/borg/archive.py                           |  6 +--
 src/borg/archiver/check_cmd.py                | 14 +++---
 src/borg/repository.py                        | 13 +++--
 src/borg/testsuite/archiver/check_cmd_test.py | 50 +++++++++++++++++--
 4 files changed, 63 insertions(+), 20 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 337b297394..55dcaed62b 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -1929,10 +1929,10 @@ def check(
                 rebuild_manifest = True
         if rebuild_manifest:
             self.manifest = self.rebuild_manifest()
-        # On Ctrl-C, skip the remaining scans.
+        # On Ctrl-C, skip any scan not yet started; a scan already running stops at its own boundary.
+        if find_lost_archives and not sig_int:
+            self.rebuild_archives_directory()
         if not sig_int:
-            if find_lost_archives:
-                self.rebuild_archives_directory()
             self.rebuild_archives(
                 match=match,
                 first=first,
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 6a3092e3f2..28e64f0e16 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -163,12 +163,14 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         description for more details about the format string).
 
         If the ``borg check`` process receives a SIGINT signal (Ctrl-C), it stops at the
-        next safe boundary (a pack boundary during the repository check, an archive boundary
-        during the archive check), leaving the repository and its chunk index in a consistent
-        state. A partial repository check (``--max-duration``) saves its progress so a later
-        partial check resumes where it stopped; a full check restarts from the beginning.
-        With ``--repair``, an interrupted archive check may leave some archives already
-        repaired and others not yet processed, so run ``borg check --repair`` again to finish.
+        next safe boundary, leaving the repository and its chunk index in a consistent state.
+        The repository check stops after the current pack; ``--verify-data`` and
+        ``--find-lost-archives`` stop after the current chunk; a ``--repair`` archive check
+        stops between whole archives. A partial repository check (``--max-duration``) saves its
+        progress so a later partial check resumes where it stopped; a full check restarts from
+        the beginning. With ``--repair``, an interrupted archive check may leave some archives
+        already repaired and others not yet processed, so run ``borg check --repair`` again to
+        finish.
 
         During a ``--repair`` run, the archive check first rebuilds the chunk index from the
         packs, and, if the key must be recovered, scans chunks for it. These phases do not yet
diff --git a/src/borg/repository.py b/src/borg/repository.py
index ec88a65f3e..a4279240e0 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1058,16 +1058,15 @@ def store_list(namespace):
         logger.info(
             f"Checked {index_files} index files ({index_errors} errors) and {pack_files} packs ({pack_errors} errors)."
         )
+        # On Ctrl-C the check stopped early, so the summary only covers the packs seen so far.
+        done, so_far = ("Interrupted", " so far") if sig_int else ("Finished", "")
         if objs_errors == 0:
-            if sig_int:
-                logger.info(f"Interrupted {mode} repository check, no problems found so far.")
-            else:
-                logger.info(f"Finished {mode} repository check, no problems found.")
+            logger.info(f"{done} {mode} repository check, no problems found{so_far}.")
         elif repair:
-            logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).")
+            logger.error(f"{done} {mode} repository check, errors found{so_far} (repository repair not implemented).")
         else:
-            logger.error(f"Finished {mode} repository check, errors found.")
-        # True means the checked objects were clean; on Ctrl-C that covers only the packs seen so far.
+            logger.error(f"{done} {mode} repository check, errors found{so_far}.")
+        # True means the checked objects were clean; --repair returns True so the caller proceeds to fix them.
         return objs_errors == 0 or repair
 
     def list(self, limit=None, marker=None):
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 5d5c21cd4a..9e847ce79f 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -110,14 +110,12 @@ def hash_then_interrupt(key):
     with Repository(archiver.repository_path, exclusive=True) as repository:
         orig_get = repository.get
         get_calls = 0
-        interrupted_after = None
 
         def get_then_interrupt(*args, **kwargs):
-            nonlocal get_calls, interrupted_after
+            nonlocal get_calls
             get_calls += 1
             if get_calls == 3:  # trip mid-loop, after 3 chunks
                 sig_int._sig_int_triggered = True
-                interrupted_after = get_calls
             return orig_get(*args, **kwargs)
 
         monkeypatch.setattr(repository, "get", get_then_interrupt)
@@ -126,7 +124,8 @@ def get_then_interrupt(*args, **kwargs):
                 ArchiveChecker().check(repository, verify_data=True, sort_by="ts", format="{archive} {time} {id}")
         finally:
             sig_int._sig_int_triggered = False
-        assert interrupted_after == 3  # the loop stopped after verifying 3 chunks
+        # verify_data breaks at the chunk it interrupted on, and the skipped scans issue no more get()s.
+        assert get_calls == 3
 
     # nothing changed, so a normal check passes.
     cmd(archiver, "check", exit_code=0)
@@ -165,6 +164,49 @@ def create_then_interrupt(self, *args, **kwargs):
     cmd(archiver, "check", exit_code=0)
 
 
+def test_check_interrupt_skips_archive_check(archivers, request, monkeypatch):
+    """A Ctrl-C during the repository check makes a full `borg check` skip the archive check. do_check
+    raises at the sig_int guard, which sits before the archive_checker.check() call, so the raise itself
+    is the skip. Exercises the do_check path (the other soft-interrupt tests call check() directly)."""
+    archiver = request.getfixturevalue(archivers)
+    if archiver.EXE:  # a class-level monkeypatch cannot reach the borg.exe subprocess
+        pytest.skip("in-process store patch does not apply to the binary")
+    check_cmd_setup(archiver)  # produces many packs
+
+    from borgstore.store import Store
+
+    orig_hash = Store.hash
+    pack_checks = []
+
+    def hash_then_interrupt(self, key):
+        result = orig_hash(self, key)
+        if key.startswith("packs/"):  # count pack checks, not the index files hashed first
+            pack_checks.append(key)
+            if len(pack_checks) == 1:  # one Ctrl-C after the first pack is checked
+                sig_int._sig_int_triggered = True
+        return result
+
+    # spy on the archive check: "Got Ctrl-C" is also raised inside ArchiveChecker.check(), so matching the
+    # message alone would not prove the skip. Recording that check() never runs is the load-bearing assertion.
+    orig_check = ArchiveChecker.check
+    archive_check_ran = False
+
+    def spy_check(self, *args, **kwargs):
+        nonlocal archive_check_ran
+        archive_check_ran = True
+        return orig_check(self, *args, **kwargs)
+
+    monkeypatch.setattr(Store, "hash", hash_then_interrupt)
+    monkeypatch.setattr(ArchiveChecker, "check", spy_check)
+    try:
+        # exec_cmd calls Archiver.run() directly; only main() maps Error to an exit code, so it propagates.
+        with pytest.raises(Error, match="Got Ctrl-C"):
+            cmd(archiver, "check", "-v")
+    finally:
+        sig_int._sig_int_triggered = False
+    assert archive_check_ran is False  # do_check raised at the sig_int guard, before archive_checker.check()
+
+
 def test_date_matching(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)

From 4944f74748f4c3dfdcb71d30d37ac472a6dd3b4b Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 19:12:41 +0200
Subject: [PATCH 044/144] chunkers: run the simplest scan kernel unless told
 otherwise

Removes automatic kernel selection. FC_K_AUTO / BZ_K_AUTO / PHTE_K_AUTO
are gone, "auto" is no longer a value, and the scan functions dispatch
exactly the kernel id they are handed. Unset means the simplest
implementation - the plain sequential loop, or the portable OpenSSL path
for the AES chunkers - on every platform; the ids are renumbered so that
one is 0.

Guessing from arch macros was wrong on both machines it was checked on,
in opposite directions (64 MiB, 21-bit mask, every byte scanned, MB/s):

  fastcdc      scalar  blockwise  neon  avx2  avx512
  Apple M3       1996       2120  4292     -       -
  Zen 4 gcc      4733        834     -  2584    2792

  buzhash64    scalar  blockwise  neon  avx2  avx512
  Apple M3       1307       2503  2446     -       -
  Zen 4 gcc      2402        767     -  1300    1334

On the M3 an "add with shifted operand" costs two cycles, so breaking the
serial chain with NEON pays 2.1x. With gcc on Zen 4 the same update folds
into a single leaq (%r9,%rdi,2), %rdi retiring in one cycle, leaving no
dependency worth breaking - so every block-parallel kernel there loses to
the sequential loop, AVX-512 included. perf confirms the dispatch rather
than the timing: the scalar run has zero samples in any SIMD kernel and
uses 1.13 G cycles against AVX-512's 1.53 G for identical output.

All kernels are kept and all stay selectable by name, so anyone who has
measured their own hardware can pin the winner. What is dropped is
borg picking one on their behalf from information that does not predict
it - the CPU alone does not, since the compiler decides whether that leaq
appears at all.

Also drops the blanks after the commas in the kernel name lists.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                              | 12 ++--
 src/borg/chunkers/buzhash64.pyx               | 19 +++---
 src/borg/chunkers/buzhash64_impl.c            | 58 ++++---------------
 src/borg/chunkers/buzhash64_impl.h            | 19 +++---
 src/borg/chunkers/fastcdc.pyx                 | 20 ++++---
 src/borg/chunkers/fastcdc_impl.c              | 44 +-------------
 src/borg/chunkers/fastcdc_impl.h              | 30 ++++++----
 src/borg/chunkers/goldilocks_aes.pyx          | 12 ++--
 src/borg/chunkers/goldilocks_aes_impl.h       |  2 +-
 src/borg/chunkers/kernel_env.py               | 24 ++++----
 src/borg/chunkers/phte_core.h                 | 16 ++---
 src/borg/chunkers/phte_kernel.h               | 18 +++---
 src/borg/chunkers/rabin_aes.pyx               |  5 +-
 src/borg/chunkers/rabin_aes_impl.h            |  2 +-
 src/borg/chunkers/toeplitz_aes.pyx            |  5 +-
 src/borg/chunkers/toeplitz_aes_impl.h         |  2 +-
 src/borg/testsuite/chunkers/buzhash64_test.py |  5 +-
 src/borg/testsuite/chunkers/fastcdc_test.py   | 28 ++++-----
 .../testsuite/chunkers/phte_chunkers_test.py  |  5 +-
 19 files changed, 131 insertions(+), 195 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 592974bc66..7f6f7ea660 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -188,12 +188,12 @@ New features:
   - fastcdc is the new and faster default chunker, #9957
   - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043:
 
-    - AVX-512 / AVX2 on x86-64 (Intel / AMD)
-    - NEON on aarch64 (e.g. Apple Silicon); the default for fastcdc, but not
-      for buzhash64, which is faster blockwise on the cores measured so far
-    - blockwise scalar elsewhere
-    - the kernel can be pinned via BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL
-      / BORG_AES_CHUNKER_KERNEL
+    - AVX-512 / AVX2 on x86-64 (Intel / AMD), NEON on aarch64, plus a portable
+      blockwise one; all bit-identical to the sequential loop
+    - the sequential loop is what runs unless BORG_FASTCDC_KERNEL /
+      BORG_BUZHASH64_KERNEL / BORG_AES_CHUNKER_KERNEL selects another kernel:
+      which one is fastest depends on the CPU *and* the compiler, so nothing
+      is chosen automatically
   - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers
     (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987
   - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64
diff --git a/src/borg/chunkers/buzhash64.pyx b/src/borg/chunkers/buzhash64.pyx
index cacee641be..7f4da08f43 100644
--- a/src/borg/chunkers/buzhash64.pyx
+++ b/src/borg/chunkers/buzhash64.pyx
@@ -21,7 +21,7 @@ cdef extern from "buzhash64_impl.h":
     const char *bz64_kernel_name(int kernel)
     int bz64_kernel_select(const char *name, int *out_id)
     const char *bz64_kernel_names()
-    int BZ_K_AUTO
+    int BZ_K_SCALAR
 
 from .kernel_env import kernel_error, requested_kernel
 
@@ -107,12 +107,16 @@ cdef uint64_t _buzhash64_update(uint64_t sum, unsigned char remove, unsigned cha
 
 
 cdef int _select_kernel() except -1:
-    """Resolve BORG_BUZHASH64_KERNEL to a kernel id, raising if it cannot be honoured."""
-    cdef int kid = BZ_K_AUTO
+    """Resolve BORG_BUZHASH64_KERNEL to a kernel id, raising if it cannot be honoured.
+
+    Unset means the simplest implementation, not the fastest one: nothing here
+    guesses which kernel a given CPU and compiler make fastest.
+    """
+    cdef int kid = BZ_K_SCALAR
     cdef int rc
     want = requested_kernel("BORG_BUZHASH64_KERNEL")
     if want is None:
-        return BZ_K_AUTO
+        return BZ_K_SCALAR
     rc = bz64_kernel_select(want.encode("ascii"), &kid)
     if rc != 0:
         raise kernel_error("BORG_BUZHASH64_KERNEL", want, rc,
@@ -176,11 +180,8 @@ cdef class ChunkerBuzHash64(ChunkerBase):
     def kernel(self):
         """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'.
 
-        'neon' exists on aarch64 but is never auto-selected (it loses to
-        'blockwise' there); BORG_BUZHASH64_KERNEL=neon selects it.
-
-        With BORG_BUZHASH64_KERNEL set to anything but "auto", this is always the
-        requested kernel - creating the chunker fails otherwise.
+        'scalar' unless BORG_BUZHASH64_KERNEL names another one, in which case
+        this is always that one - creating the chunker fails otherwise.
         """
         return (<bytes>bz64_kernel_name(self.kernel_id)).decode("ascii")
 
diff --git a/src/borg/chunkers/buzhash64_impl.c b/src/borg/chunkers/buzhash64_impl.c
index 1f24d4eedd..180bb58394 100644
--- a/src/borg/chunkers/buzhash64_impl.c
+++ b/src/borg/chunkers/buzhash64_impl.c
@@ -114,12 +114,13 @@ static size_t bz64_scan_blockwise(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
-/* --- NEON (aarch64, selectable but not the default) ---------------------
+/* --- NEON (aarch64) ------------------------------------------------------
  *
- * This kernel is NOT auto-selected: on an Apple M3 Pro it loses to the
- * blockwise scalar one (2590 vs 2360 MB/s, the same ~9% gap at every mask
- * size from 17 to 23 bits), so bz64_simd_available() returns 0 here and
- * BZ_K_AUTO resolves to blockwise. BORG_BUZHASH64_KERNEL=neon selects it.
+ * Nothing selects this by default (the default is the sequential kernel
+ * everywhere); BORG_BUZHASH64_KERNEL=neon selects it. On an Apple M3 Pro it
+ * loses to the blockwise kernel - 2590 vs 2360 MB/s, the same ~9% gap at
+ * every mask size from 17 to 23 bits - so it is not the one to reach for
+ * there.
  *
  * Why it loses there: the 16 table lookups per block have to happen in
  * general registers (NEON has no gather), so the vector form only ADDS the
@@ -136,8 +137,7 @@ static size_t bz64_scan_blockwise(const uint64_t *T, const uint64_t *Trot,
  * comparatively healthy NEON, which is exactly where this should get
  * competitive - on this machine's much narrower E-cores the 9% gap already
  * collapses into measurement noise. If you have such hardware, compare
- * BORG_BUZHASH64_KERNEL=neon against =blockwise and please report; flipping
- * the default is then a one-line change here. */
+ * BORG_BUZHASH64_KERNEL=neon against =blockwise and please report. */
 
 #if defined(__aarch64__)
 #define BZ_KIND "neon"
@@ -179,10 +179,6 @@ static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
-static int bz64_simd_available(void)
-{
-    return 0; /* see above: selectable by name, but never auto-selected */
-}
 
 /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */
 
@@ -348,14 +344,6 @@ bz64_scan_simd512(const uint64_t *T, const uint64_t *Trot,
     return j;
 }
 
-static int bz64_simd_available(void)
-{
-#ifdef BZ_KIND_512
-    if (__builtin_cpu_supports("avx512f"))
-        return 2;
-#endif
-    return __builtin_cpu_supports("avx2");
-}
 
 #else
 #define BZ_KIND "blockwise"
@@ -367,10 +355,6 @@ static size_t bz64_scan_simd(const uint64_t *T, const uint64_t *Trot,
     return bz64_scan_blockwise(T, Trot, pr, pa, n, sum_io, mask);
 }
 
-static int bz64_simd_available(void)
-{
-    return 0;
-}
 
 #endif
 
@@ -379,20 +363,16 @@ static int bz64_simd_available(void)
 const char *bz64_kernel_names(void)
 {
 #if defined(__aarch64__)
-    return "auto, neon, blockwise, scalar";
+    return "neon,blockwise,scalar";
 #elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
-    return "auto, avx512, avx2, blockwise, scalar";
+    return "avx512,avx2,blockwise,scalar";
 #else
-    return "auto, blockwise, scalar";
+    return "blockwise,scalar";
 #endif
 }
 
 int bz64_kernel_select(const char *name, int *out_id)
 {
-    if (strcmp(name, "auto") == 0) {
-        *out_id = BZ_K_AUTO;
-        return BZ_KSEL_OK;
-    }
     if (strcmp(name, "scalar") == 0) {
         *out_id = BZ_K_SCALAR;
         return BZ_KSEL_OK;
@@ -429,26 +409,10 @@ int bz64_kernel_select(const char *name, int *out_id)
 
 /* --- dispatch ----------------------------------------------------------- */
 
-/* the kernel BZ_K_AUTO resolves to, worked out once; a racy double-init
- * writes the same value, so it is benign */
-static int bz64_auto = -1;
-
-static int bz64_auto_kernel(void)
-{
-    int a;
-    if (bz64_auto < 0) {
-        a = bz64_simd_available();
-        bz64_auto = (a == 2) ? BZ_K_VECTOR512 : (a ? BZ_K_VECTOR : BZ_K_BLOCKWISE);
-    }
-    return bz64_auto;
-}
-
 size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                  const uint8_t *p_rem, const uint8_t *p_add,
                  size_t n, uint64_t *sum, uint64_t mask, int kernel)
 {
-    if (kernel == BZ_K_AUTO)
-        kernel = bz64_auto_kernel();
     switch (kernel) {
     case BZ_K_SCALAR:
         return bz64_scan_seq(table, table_rot, p_rem, p_add, n, sum, mask);
@@ -465,8 +429,6 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
 
 const char *bz64_kernel_name(int kernel)
 {
-    if (kernel == BZ_K_AUTO)
-        kernel = bz64_auto_kernel();
     switch (kernel) {
     case BZ_K_SCALAR:
         return "scalar";
diff --git a/src/borg/chunkers/buzhash64_impl.h b/src/borg/chunkers/buzhash64_impl.h
index c75496ce35..d15b5e0ebb 100644
--- a/src/borg/chunkers/buzhash64_impl.h
+++ b/src/borg/chunkers/buzhash64_impl.h
@@ -21,14 +21,14 @@
  * ROTL(table[b], window_size % 64) (precomputed by the caller).
  * p_rem points at the byte leaving the window, p_add at the byte entering
  * (p_add = p_rem + window_size); both must have n readable bytes.
- * kernel is one of BZ_K_*; BZ_K_AUTO picks the best one this CPU can run.
+ * kernel is one of BZ_K_*.
  * All kernels return bit-identical results. */
-/* Scan kernel ids, a tier ladder; see fastcdc_impl.h for the rationale. */
-#define BZ_K_AUTO 0      /* best kernel this CPU can run */
-#define BZ_K_SCALAR 1    /* sequential reference loop */
-#define BZ_K_BLOCKWISE 2 /* portable 8-lane C */
-#define BZ_K_VECTOR 3    /* neon / avx2 (neon is not auto-selected) */
-#define BZ_K_VECTOR512 4 /* avx512 */
+/* Scan kernel ids; see fastcdc_impl.h for the rationale. There is no
+ * automatic selection - the caller says which kernel to run. */
+#define BZ_K_SCALAR 0    /* sequential reference loop */
+#define BZ_K_BLOCKWISE 1 /* portable 8-lane C */
+#define BZ_K_VECTOR 2    /* neon / avx2 (neon is not auto-selected) */
+#define BZ_K_VECTOR512 3 /* avx512 */
 
 /* Results of bz64_kernel_select(). */
 #define BZ_KSEL_OK 0
@@ -46,9 +46,8 @@ size_t bz64_scan(const uint64_t *table, const uint64_t *table_rot,
                  const uint8_t *p_rem, const uint8_t *p_add,
                  size_t n, uint64_t *sum, uint64_t mask, int kernel);
 
-/* Name of the kernel <kernel> selects: "neon", "avx512", "avx2", "blockwise"
- * or "scalar"; for BZ_K_AUTO, the auto-selected one. Note "neon" is never
- * auto-selected, see buzhash64_impl.c. */
+/* Name of the kernel <kernel> selects: "neon", "avx512", "avx2",
+ * "blockwise" or "scalar". */
 const char *bz64_kernel_name(int kernel);
 
 #endif
diff --git a/src/borg/chunkers/fastcdc.pyx b/src/borg/chunkers/fastcdc.pyx
index ca0d11216c..b72eefea4c 100644
--- a/src/borg/chunkers/fastcdc.pyx
+++ b/src/borg/chunkers/fastcdc.pyx
@@ -17,7 +17,7 @@ cdef extern from "fastcdc_impl.h":
     const char *fc_kernel_name(int kernel)
     int fc_kernel_select(const char *name, int *out_id)
     const char *fc_kernel_names()
-    int FC_K_AUTO
+    int FC_K_SCALAR
 
 from .kernel_env import kernel_error, requested_kernel
 
@@ -40,8 +40,8 @@ from .kernel_env import kernel_error, requested_kernel
 # The inner scan runs in a C kernel (fastcdc_impl.c) with SIMD implementations (NEON on
 # aarch64, AVX-512 or AVX2 on x86-64, blockwise scalar elsewhere) that are bit-identical to
 # the plain sequential Gear loop: every byte position is tested, cuts and hash state are
-# exactly the same, only faster. BORG_FASTCDC_KERNEL pins the kernel instead of letting it
-# be auto-selected; see kernel_env.py and the .kernel property.
+# exactly the same, only faster. The sequential loop is what runs by default; the others
+# are selected via BORG_FASTCDC_KERNEL, see kernel_env.py and the .kernel property.
 
 
 @cython.boundscheck(False)
@@ -65,12 +65,16 @@ cdef uint64_t* fastcdc_init_gear(bytes key) except NULL:
 
 
 cdef int _select_kernel() except -1:
-    """Resolve BORG_FASTCDC_KERNEL to a kernel id, raising if it cannot be honoured."""
-    cdef int kid = FC_K_AUTO
+    """Resolve BORG_FASTCDC_KERNEL to a kernel id, raising if it cannot be honoured.
+
+    Unset means the simplest implementation, not the fastest one: nothing here
+    guesses which kernel a given CPU and compiler make fastest.
+    """
+    cdef int kid = FC_K_SCALAR
     cdef int rc
     want = requested_kernel("BORG_FASTCDC_KERNEL")
     if want is None:
-        return FC_K_AUTO
+        return FC_K_SCALAR
     rc = fc_kernel_select(want.encode("ascii"), &kid)
     if rc != 0:
         raise kernel_error("BORG_FASTCDC_KERNEL", want, rc,
@@ -107,8 +111,8 @@ cdef class ChunkerFastCDC(ChunkerBase):
     def kernel(self):
         """Which scan kernel this chunker uses: 'neon', 'avx512', 'avx2', 'blockwise' or 'scalar'.
 
-        With BORG_FASTCDC_KERNEL set to anything but "auto", this is always the
-        requested kernel - creating the chunker fails otherwise.
+        'scalar' unless BORG_FASTCDC_KERNEL names another one, in which case
+        this is always that one - creating the chunker fails otherwise.
         """
         return (<bytes>fc_kernel_name(self.kernel_id)).decode("ascii")
 
diff --git a/src/borg/chunkers/fastcdc_impl.c b/src/borg/chunkers/fastcdc_impl.c
index ac58923727..2738dc72f8 100644
--- a/src/borg/chunkers/fastcdc_impl.c
+++ b/src/borg/chunkers/fastcdc_impl.c
@@ -172,10 +172,6 @@ static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, ui
     return -1;
 }
 
-static int fc_simd_available(void)
-{
-    return 1;
-}
 
 /* --- AVX2 (x86-64, runtime detected) ------------------------------------ */
 
@@ -342,14 +338,6 @@ fc_scan_simd512(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp_i
     return -1;
 }
 
-static int fc_simd_available(void)
-{
-#ifdef FC_KIND_512
-    if (__builtin_cpu_supports("avx512f"))
-        return 2;
-#endif
-    return __builtin_cpu_supports("avx2");
-}
 
 #else
 #define FC_KIND "blockwise"
@@ -359,10 +347,6 @@ static int64_t fc_scan_simd(const uint64_t *gear, const uint8_t *p, size_t n, ui
     return fc_scan_blockwise(gear, p, n, fp_io, mask);
 }
 
-static int fc_simd_available(void)
-{
-    return 0;
-}
 
 #endif
 
@@ -371,20 +355,16 @@ static int fc_simd_available(void)
 const char *fc_kernel_names(void)
 {
 #if defined(__aarch64__)
-    return "auto, neon, blockwise, scalar";
+    return "neon,blockwise,scalar";
 #elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
-    return "auto, avx512, avx2, blockwise, scalar";
+    return "avx512,avx2,blockwise,scalar";
 #else
-    return "auto, blockwise, scalar";
+    return "blockwise,scalar";
 #endif
 }
 
 int fc_kernel_select(const char *name, int *out_id)
 {
-    if (strcmp(name, "auto") == 0) {
-        *out_id = FC_K_AUTO;
-        return FC_KSEL_OK;
-    }
     if (strcmp(name, "scalar") == 0) {
         *out_id = FC_K_SCALAR;
         return FC_KSEL_OK;
@@ -421,24 +401,8 @@ int fc_kernel_select(const char *name, int *out_id)
 
 /* --- dispatch ----------------------------------------------------------- */
 
-/* the kernel FC_K_AUTO resolves to, worked out once; a racy double-init
- * writes the same value, so it is benign */
-static int fc_auto = -1;
-
-static int fc_auto_kernel(void)
-{
-    int a;
-    if (fc_auto < 0) {
-        a = fc_simd_available();
-        fc_auto = (a == 2) ? FC_K_VECTOR512 : (a ? FC_K_VECTOR : FC_K_BLOCKWISE);
-    }
-    return fc_auto;
-}
-
 int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel)
 {
-    if (kernel == FC_K_AUTO)
-        kernel = fc_auto_kernel();
     switch (kernel) {
     case FC_K_SCALAR:
         return fc_scan_seq(gear, p, n, fp, mask);
@@ -455,8 +419,6 @@ int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp,
 
 const char *fc_kernel_name(int kernel)
 {
-    if (kernel == FC_K_AUTO)
-        kernel = fc_auto_kernel();
     switch (kernel) {
     case FC_K_SCALAR:
         return "scalar";
diff --git a/src/borg/chunkers/fastcdc_impl.h b/src/borg/chunkers/fastcdc_impl.h
index 145127b485..7b637908b8 100644
--- a/src/borg/chunkers/fastcdc_impl.h
+++ b/src/borg/chunkers/fastcdc_impl.h
@@ -11,13 +11,20 @@
 #include <stdint.h>
 #include <stddef.h>
 
-/* Scan kernel ids, a tier ladder. Which names map onto them depends on the
- * build: "neon" exists only on aarch64, "avx2"/"avx512" only on x86-64. */
-#define FC_K_AUTO 0      /* best kernel this CPU can run */
-#define FC_K_SCALAR 1    /* sequential reference loop */
-#define FC_K_BLOCKWISE 2 /* portable 8-lane C */
-#define FC_K_VECTOR 3    /* the platform's vector kernel: neon or avx2 */
-#define FC_K_VECTOR512 4 /* avx512 */
+/* Scan kernel ids. Which names map onto them depends on the build: "neon"
+ * exists only on aarch64, "avx2"/"avx512" only on x86-64.
+ *
+ * There is no automatic selection: the caller says which kernel to run, and
+ * that is what runs. FC_K_SCALAR, the plain sequential loop, is id 0 and the
+ * default everywhere - it is the simplest implementation and the one the
+ * others are checked against. Which of the rest is fastest turned out not to
+ * be predictable from the instruction set (on an Apple M3 the NEON kernel
+ * beats the sequential loop 2.1x; with gcc on a Zen 4 the sequential loop
+ * beats AVX-512 by 1.7x), so choosing one is left to whoever measured. */
+#define FC_K_SCALAR 0    /* sequential reference loop */
+#define FC_K_BLOCKWISE 1 /* portable 8-lane C */
+#define FC_K_VECTOR 2    /* the platform's vector kernel: neon or avx2 */
+#define FC_K_VECTOR512 3 /* avx512 */
 
 /* Results of fc_kernel_select(). */
 #define FC_KSEL_OK 0
@@ -39,14 +46,13 @@ const char *fc_kernel_names(void);
  * and test (fp & mask) == 0.
  * Returns the first i that matched (fp is left at position i), or -1 if none
  * matched (fp is left at position n-1).
- * gear: the keyed 256-entry table. kernel is one of FC_K_*; FC_K_AUTO picks
- * the best one this CPU can run. Callers are expected to have validated any
- * explicit choice with fc_kernel_select() - an unrunnable one falls back to
- * FC_K_AUTO rather than crashing.
+ * gear: the keyed 256-entry table. kernel is one of FC_K_*; callers are
+ * expected to have validated it with fc_kernel_select(), an unrunnable one
+ * falls back to the vector kernel rather than crashing.
  * All kernels return bit-identical results. */
 int64_t fc_scan(const uint64_t *gear, const uint8_t *p, size_t n, uint64_t *fp, uint64_t mask, int kernel);
 
-/* Name of the kernel <kernel> selects; for FC_K_AUTO, the auto-selected one. */
+/* Name of the kernel <kernel> selects. */
 const char *fc_kernel_name(int kernel);
 
 #endif
diff --git a/src/borg/chunkers/goldilocks_aes.pyx b/src/borg/chunkers/goldilocks_aes.pyx
index 329577a723..4c86a118c0 100644
--- a/src/borg/chunkers/goldilocks_aes.pyx
+++ b/src/borg/chunkers/goldilocks_aes.pyx
@@ -48,8 +48,8 @@ cdef extern from "goldilocks_aes_impl.h":
     GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t *aes_key, int kernel)
     int phte_kernel_select(const char *name, int *out_id)
     const char *phte_kernel_names()
-    int PHTE_K_AUTO
     int PHTE_K_EVP
+    int PHTE_K_HW
     void gl_free(GL_CTX *ctx)
     const char *gl_kind(const GL_CTX *ctx)
     uint64_t gl_digest64(const GL_CTX *ctx, const uint8_t *q)
@@ -118,11 +118,11 @@ cdef int _select_kernel() except -1:
     One selector for all three AES chunkers: they share phte_scan.h, so the
     available paths never differ between them.
     """
-    cdef int kid = PHTE_K_AUTO
+    cdef int kid = PHTE_K_EVP
     cdef int rc
     want = requested_kernel("BORG_AES_CHUNKER_KERNEL")
     if want is None:
-        return PHTE_K_AUTO
+        return PHTE_K_EVP
     rc = phte_kernel_select(want.encode("ascii"), &kid)
     if rc != 0:
         raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc,
@@ -214,7 +214,9 @@ def goldilocks_aes_scan_all(k, bytes aes_key, bytes data, mask, bint force_sw=Fa
 
     Returns the list of positions i where the cut condition matches, i.e.
     (AES(state of window data[i-63..i]) & mask) == 0 - independent of any
-    chunk min/max framing. force_sw selects the portable EVP path.
+    chunk min/max framing. force_sw selects the portable EVP path, otherwise
+    the AES hardware path is used (which falls back to EVP where the CPU has
+    no AES instructions).
     """
     cdef uint64_t c_tables[3 * 256]
     cdef GL_CTX *ctx
@@ -230,7 +232,7 @@ def goldilocks_aes_scan_all(k, bytes aes_key, bytes data, mask, bint force_sw=Fa
     for t in range(len(tables)):
         for i in range(256):
             c_tables[t * 256 + i] = tables[t][i]
-    ctx = gl_new(c_tables, k, (k * k) % _GL_P, <const uint8_t*>PyBytes_AsString(aes_key), PHTE_K_EVP if force_sw else PHTE_K_AUTO)
+    ctx = gl_new(c_tables, k, (k * k) % _GL_P, <const uint8_t*>PyBytes_AsString(aes_key), PHTE_K_EVP if force_sw else PHTE_K_HW)
     if ctx == NULL:
         raise MemoryError("Failed to set up goldilocks-aes kernel")
     try:
diff --git a/src/borg/chunkers/goldilocks_aes_impl.h b/src/borg/chunkers/goldilocks_aes_impl.h
index f1f04a3444..d8c61d1044 100644
--- a/src/borg/chunkers/goldilocks_aes_impl.h
+++ b/src/borg/chunkers/goldilocks_aes_impl.h
@@ -39,7 +39,7 @@ typedef struct GL_CTX GL_CTX;
  * k1: the secret evaluation point K (canonical, 0 <= K < p).
  * k2: K^2 mod p.
  * aes_key: 16 bytes (AES-128).
- * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
+ * kernel: one of PHTE_K_*.
  * Returns NULL on allocation/OpenSSL failure. */
 GL_CTX *gl_new(const uint64_t *tables, uint64_t k1, uint64_t k2, const uint8_t aes_key[16], int kernel);
 
diff --git a/src/borg/chunkers/kernel_env.py b/src/borg/chunkers/kernel_env.py
index aacfcc76ae..cac22ef2c5 100644
--- a/src/borg/chunkers/kernel_env.py
+++ b/src/borg/chunkers/kernel_env.py
@@ -1,14 +1,16 @@
 """Shared parsing for the BORG_*_KERNEL scan-kernel selection env vars.
 
-Each SIMD-accelerated chunker picks its inner scan kernel automatically. The
-env vars exist to pin it instead - for benchmarking one kernel against
-another, and for CI that wants to prove a particular kernel was exercised
-rather than hope it was.
-
-"auto" (the default) takes the best kernel the CPU can run. Any other value is
-a demand, not a preference: if that kernel cannot run here, chunker creation
-raises ValueError rather than quietly falling back, because a silent fallback
-turns a benchmark or a test into a measurement of something else.
+Each chunker runs the simplest implementation - the plain sequential loop, or
+the portable OpenSSL path for the AES chunkers - unless one of these env vars
+names a different one. There is no automatic selection: which kernel is
+fastest turned out not to be predictable from the instruction set (an Apple M3
+runs NEON 2.1x faster than the sequential loop; gcc on a Zen 4 runs the
+sequential loop 1.7x faster than AVX-512), so nothing guesses on the user's
+behalf.
+
+A value is a demand, not a preference: if that kernel cannot run here, chunker
+creation raises ValueError rather than quietly falling back, because a silent
+fallback turns a benchmark or a test into a measurement of something else.
 
 The C side resolves names, since which names exist depends on the build and
 the CPU; this module only turns its verdict into an exception.
@@ -37,6 +39,6 @@ def kernel_error(envvar, want, status, valid):
 
 
 def requested_kernel(envvar):
-    """The kernel name requested via <envvar>, or None for auto/unset."""
+    """The kernel name requested via <envvar>, or None if unset (use the default)."""
     want = os.environ.get(envvar, "").strip().lower()
-    return None if want in ("", "auto") else want
+    return want or None
diff --git a/src/borg/chunkers/phte_core.h b/src/borg/chunkers/phte_core.h
index 7c7925e698..13be52eb49 100644
--- a/src/borg/chunkers/phte_core.h
+++ b/src/borg/chunkers/phte_core.h
@@ -165,20 +165,16 @@ static int phte_hw512_available(void)
 const char *phte_kernel_names(void)
 {
 #if defined(__aarch64__)
-    return "auto, aes-arm64, evp";
+    return "aes-arm64,evp";
 #elif (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__))
-    return "auto, vaes, aes-ni, evp";
+    return "vaes,aes-ni,evp";
 #else
-    return "auto, evp";
+    return "evp";
 #endif
 }
 
 int phte_kernel_select(const char *name, int *out_id)
 {
-    if (strcmp(name, "auto") == 0) {
-        *out_id = PHTE_K_AUTO;
-        return PHTE_KSEL_OK;
-    }
     if (strcmp(name, "evp") == 0) {
         *out_id = PHTE_K_EVP;
         return PHTE_KSEL_OK;
@@ -209,18 +205,18 @@ int phte_kernel_select(const char *name, int *out_id)
 /* --- context base management ------------------------------------------- */
 
 /* Expand the AES key, select the scan path and set up the OpenSSL context.
- * kernel is one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
+ * kernel is one of PHTE_K_*.
  * Returns 0 on failure (caller frees its context). */
 static int phte_base_init(PHTE_BASE *b, const uint8_t aes_key[16], int kernel)
 {
     phte_aes128_expand(aes_key, b->rk);
 #if PHTE_HAVE_HW
-    b->use_hw = (kernel == PHTE_K_AUTO || kernel == PHTE_K_HW || kernel == PHTE_K_HW512) && phte_hw_available();
+    b->use_hw = (kernel == PHTE_K_HW || kernel == PHTE_K_HW512) && phte_hw_available();
 #else
     b->use_hw = 0;
 #endif
 #if PHTE_HAVE_HW512
-    b->use_hw512 = b->use_hw && kernel != PHTE_K_HW && phte_hw512_available();
+    b->use_hw512 = b->use_hw && kernel == PHTE_K_HW512 && phte_hw512_available();
 #else
     b->use_hw512 = 0;
 #endif
diff --git a/src/borg/chunkers/phte_kernel.h b/src/borg/chunkers/phte_kernel.h
index 1340c55d34..9ec1fa1e2b 100644
--- a/src/borg/chunkers/phte_kernel.h
+++ b/src/borg/chunkers/phte_kernel.h
@@ -9,13 +9,17 @@
 #ifndef BORG_PHTE_KERNEL_H
 #define BORG_PHTE_KERNEL_H
 
-/* Scan path ids, a tier ladder. Which names map onto them depends on the
- * build: "vaes" only on x86-64, and the 128-bit hardware path is spelled
- * "aes-ni" on x86-64 and "aes-arm64" on aarch64. */
-#define PHTE_K_AUTO 0  /* best path this CPU can run */
-#define PHTE_K_EVP 1   /* portable OpenSSL EVP batch path */
-#define PHTE_K_HW 2    /* 128-bit AES instructions: aes-ni / aes-arm64 */
-#define PHTE_K_HW512 3 /* VAES/AVX-512, 4 AES blocks per instruction */
+/* Scan path ids. Which names map onto them depends on the build: "vaes" only
+ * on x86-64, and the 128-bit hardware path is spelled "aes-ni" on x86-64 and
+ * "aes-arm64" on aarch64.
+ *
+ * There is no automatic selection: the caller says which path to run.
+ * PHTE_K_EVP, the portable OpenSSL batch path, is id 0 and the default - it
+ * is the simplest implementation and the one the others are checked
+ * against. */
+#define PHTE_K_EVP 0   /* portable OpenSSL EVP batch path */
+#define PHTE_K_HW 1    /* 128-bit AES instructions: aes-ni / aes-arm64 */
+#define PHTE_K_HW512 2 /* VAES/AVX-512, 4 AES blocks per instruction */
 
 /* Results of phte_kernel_select(). */
 #define PHTE_KSEL_OK 0
diff --git a/src/borg/chunkers/rabin_aes.pyx b/src/borg/chunkers/rabin_aes.pyx
index fa55ded145..953d73c444 100644
--- a/src/borg/chunkers/rabin_aes.pyx
+++ b/src/borg/chunkers/rabin_aes.pyx
@@ -60,7 +60,6 @@ cdef extern from "rabin_aes_impl.h":
     RA_CTX *ra_new(const uint64_t *tables, const uint8_t *aes_key, int kernel)
     int phte_kernel_select(const char *name, int *out_id)
     const char *phte_kernel_names()
-    int PHTE_K_AUTO
     int PHTE_K_EVP
     void ra_free(RA_CTX *ctx)
     const char *ra_kind(const RA_CTX *ctx)
@@ -188,11 +187,11 @@ cdef int _select_kernel() except -1:
     One selector for all three AES chunkers: they share phte_scan.h, so the
     available paths never differ between them.
     """
-    cdef int kid = PHTE_K_AUTO
+    cdef int kid = PHTE_K_EVP
     cdef int rc
     want = requested_kernel("BORG_AES_CHUNKER_KERNEL")
     if want is None:
-        return PHTE_K_AUTO
+        return PHTE_K_EVP
     rc = phte_kernel_select(want.encode("ascii"), &kid)
     if rc != 0:
         raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc,
diff --git a/src/borg/chunkers/rabin_aes_impl.h b/src/borg/chunkers/rabin_aes_impl.h
index 5a4fc07e40..bb3755f4d6 100644
--- a/src/borg/chunkers/rabin_aes_impl.h
+++ b/src/borg/chunkers/rabin_aes_impl.h
@@ -36,7 +36,7 @@ typedef struct RA_CTX RA_CTX;
 /* Create a kernel context.
  * tables: RA_TABLES * 256 uint64 entries, see above.
  * aes_key: 16 bytes (AES-128).
- * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
+ * kernel: one of PHTE_K_*.
  * Returns NULL on allocation/OpenSSL failure. */
 RA_CTX *ra_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel);
 
diff --git a/src/borg/chunkers/toeplitz_aes.pyx b/src/borg/chunkers/toeplitz_aes.pyx
index 8bc5ff423c..f9fc7cbb44 100644
--- a/src/borg/chunkers/toeplitz_aes.pyx
+++ b/src/borg/chunkers/toeplitz_aes.pyx
@@ -53,7 +53,6 @@ cdef extern from "toeplitz_aes_impl.h":
     TP_CTX *tp_new(const uint64_t *tables, const uint8_t *aes_key, int kernel)
     int phte_kernel_select(const char *name, int *out_id)
     const char *phte_kernel_names()
-    int PHTE_K_AUTO
     int PHTE_K_EVP
     void tp_free(TP_CTX *ctx)
     const char *tp_kind(const TP_CTX *ctx)
@@ -120,11 +119,11 @@ cdef int _select_kernel() except -1:
     One selector for all three AES chunkers: they share phte_scan.h, so the
     available paths never differ between them.
     """
-    cdef int kid = PHTE_K_AUTO
+    cdef int kid = PHTE_K_EVP
     cdef int rc
     want = requested_kernel("BORG_AES_CHUNKER_KERNEL")
     if want is None:
-        return PHTE_K_AUTO
+        return PHTE_K_EVP
     rc = phte_kernel_select(want.encode("ascii"), &kid)
     if rc != 0:
         raise kernel_error("BORG_AES_CHUNKER_KERNEL", want, rc,
diff --git a/src/borg/chunkers/toeplitz_aes_impl.h b/src/borg/chunkers/toeplitz_aes_impl.h
index 5aa3f61f8b..124b5df2fc 100644
--- a/src/borg/chunkers/toeplitz_aes_impl.h
+++ b/src/borg/chunkers/toeplitz_aes_impl.h
@@ -40,7 +40,7 @@ typedef struct TP_CTX TP_CTX;
 /* Create a kernel context.
  * tables: TP_TABLES * 256 uint64 entries, see above.
  * aes_key: 16 bytes (AES-128).
- * kernel: one of PHTE_K_*; PHTE_K_AUTO picks the best path this CPU can run.
+ * kernel: one of PHTE_K_*.
  * Returns NULL on allocation/OpenSSL failure. */
 TP_CTX *tp_new(const uint64_t *tables, const uint8_t aes_key[16], int kernel);
 
diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py
index 0848e8f9be..202d32ed81 100644
--- a/src/borg/testsuite/chunkers/buzhash64_test.py
+++ b/src/borg/testsuite/chunkers/buzhash64_test.py
@@ -142,7 +142,7 @@ def rnd_key():
             assert reconstructed == data
 
 
-ALL_BUZHASH64_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar")
+ALL_BUZHASH64_KERNELS = ("neon", "avx512", "avx2", "blockwise", "scalar")
 
 
 def test_buzhash64_kernels_identical(monkeypatch):
@@ -165,8 +165,7 @@ def sizes(chunker):
             chunker = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
         except ValueError:
             continue  # not available on this build/CPU
-        if name != "auto":
-            assert chunker.kernel == name
+        assert chunker.kernel == name
         assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one"
         tested.append(name)
 
diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py
index 9e19870ad9..1a7eaaf2e3 100644
--- a/src/borg/testsuite/chunkers/fastcdc_test.py
+++ b/src/borg/testsuite/chunkers/fastcdc_test.py
@@ -178,7 +178,7 @@ def rnd_key():
             assert b"".join(parts) == data
 
 
-ALL_FASTCDC_KERNELS = ("auto", "neon", "avx512", "avx2", "blockwise", "scalar")
+ALL_FASTCDC_KERNELS = ("neon", "avx512", "avx2", "blockwise", "scalar")
 
 
 def test_fastcdc_kernels_identical(monkeypatch):
@@ -202,8 +202,7 @@ def sizes(chunker):
             chunker = ChunkerFastCDC(key0, 10, 16, 14, 2)
         except ValueError:
             continue  # not available on this build/CPU
-        if name != "auto":
-            assert chunker.kernel == name
+        assert chunker.kernel == name
         assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one"
         tested.append(name)
 
@@ -218,10 +217,10 @@ def test_kernel_env_rejects_unusable(envvar, monkeypatch):
     # means to pin one kernel, into a measurement of a different one.
     from ...chunkers import ChunkerBuzHash64, ChunkerToeplitzAES
 
-    make = {
-        "BORG_FASTCDC_KERNEL": lambda k: ChunkerFastCDC(k, 10, 16, 14, 2),
-        "BORG_BUZHASH64_KERNEL": lambda k: ChunkerBuzHash64(k, 10, 16, 14, 4095, 2),
-        "BORG_AES_CHUNKER_KERNEL": lambda k: ChunkerToeplitzAES(k, 10, 16, 14, 2),
+    make, default = {
+        "BORG_FASTCDC_KERNEL": (lambda k: ChunkerFastCDC(k, 10, 16, 14, 2), "scalar"),
+        "BORG_BUZHASH64_KERNEL": (lambda k: ChunkerBuzHash64(k, 10, 16, 14, 4095, 2), "scalar"),
+        "BORG_AES_CHUNKER_KERNEL": (lambda k: ChunkerToeplitzAES(k, 10, 16, 14, 2), "evp"),
     }[envvar]
     key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
 
@@ -229,12 +228,15 @@ def test_kernel_env_rejects_unusable(envvar, monkeypatch):
     with pytest.raises(ValueError, match="no-such-kernel"):
         make(key0)
 
-    # "auto" and an unset var both mean "pick the best one", and always work
+    # there is no automatic selection, so "auto" is not a kernel name either
     monkeypatch.setenv(envvar, "auto")
-    auto = make(key0).kernel
+    with pytest.raises(ValueError, match="auto"):
+        make(key0)
+
+    # unset means the simplest implementation, on every platform
     monkeypatch.delenv(envvar)
-    assert make(key0).kernel == auto
+    assert make(key0).kernel == default
 
-    # asking for whatever auto-selection picked must yield exactly that
-    monkeypatch.setenv(envvar, auto)
-    assert make(key0).kernel == auto
+    # and asking for it explicitly gives the same thing
+    monkeypatch.setenv(envvar, default)
+    assert make(key0).kernel == default
diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py
index 19b89ffac7..a231df6002 100644
--- a/src/borg/testsuite/chunkers/phte_chunkers_test.py
+++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py
@@ -44,7 +44,7 @@ def chunker_spec(request):
     return request.param
 
 
-ALL_AES_KERNELS = ("auto", "vaes", "aes-ni", "aes-arm64", "evp")
+ALL_AES_KERNELS = ("vaes", "aes-ni", "aes-arm64", "evp")
 
 
 def test_kernels_identical(chunker_spec, monkeypatch):
@@ -67,8 +67,7 @@ def sizes(chunker):
             chunker = cls(key0, 10, 16, 14, 2)
         except ValueError:
             continue  # not available on this build/CPU
-        if name != "auto":
-            assert chunker.kernel == name
+        assert chunker.kernel == name
         assert sizes(chunker) == reference, f"path {name} disagrees with the default one"
         tested.append(name)
 

From 1284c474b9d166f4a9a2592a1b4a21ec650b8136 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 19:12:55 +0200
Subject: [PATCH 045/144] chunkers: log the chunker and its scan kernel at
 debug level

Which scan kernel runs is not implied by the platform any more - it is
the simplest one unless BORG_*_KERNEL says otherwise - so there was no
way to see what a given run actually used. get_chunker() now reports it:

  chunker: buzhash, scan kernel: n/a (single implementation)
  chunker: fastcdc, scan kernel: scalar
  chunker: toeplitz-aes, scan kernel: aes-arm64

Chunkers with only one implementation say so rather than leaving the
field out, so that a missing kernel cannot be read as a reporting bug.

get_chunker() is the single funnel every chunker goes through, which is
why the logging sits there rather than in each of the five .pyx files.

Guarded on logger.configured: chunkers get built before setup_logging()
has run - the test suite does it, and so does anything using borg as a
library - and create_logger() raises instead of logging at that point.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/chunkers/__init__.py | 41 ++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/src/borg/chunkers/__init__.py b/src/borg/chunkers/__init__.py
index af6ded2dd3..d6f66a6cb6 100644
--- a/src/borg/chunkers/__init__.py
+++ b/src/borg/chunkers/__init__.py
@@ -8,6 +8,31 @@
 from .fixed import ChunkerFixed
 from .reader import *  # noqa
 
+from .. import logger as _logger
+
+logger = _logger.create_logger()
+
+
+def _made(algo, chunker):
+    """Log which chunker was constructed and which scan kernel it ended up with.
+
+    The kernel is worth seeing because it is not implied by the platform: it
+    defaults to the simplest implementation and is only changed by the
+    BORG_*_KERNEL env vars (see kernel_env.py).
+
+    Chunkers that have nothing to select - buzhash and fixed have exactly one
+    implementation each - say so rather than omitting the field, so that a
+    missing kernel cannot be mistaken for a reporting bug.
+
+    Chunkers can be built before setup_logging() has run (tests do it, and so
+    does anything using borg as a library), and create_logger() raises rather
+    than log at that point - so say nothing when there is nowhere to say it.
+    """
+    if _logger.configured:
+        kernel = getattr(chunker, "kernel", None) or "n/a (single implementation)"
+        logger.debug("chunker: %s, scan kernel: %s", algo, kernel)
+    return chunker
+
 
 def get_chunker(algo, *params, **kw):
     key = kw.get("key", None)
@@ -15,7 +40,7 @@ def get_chunker(algo, *params, **kw):
     # key.chunk_seed only has 32 bits
     seed = key.chunk_seed if key is not None else 0
     if algo == "buzhash":
-        return Chunker(seed, *params, sparse=sparse)
+        return _made(algo, Chunker(seed, *params, sparse=sparse))
     if algo == "buzhash64":
         # for buzhash64, we want a much longer key, so we derive it from the id key.
         # params is (chunk_min_exp, chunk_max_exp, hash_mask_bits, hash_window_size, nc_level);
@@ -23,14 +48,14 @@ def get_chunker(algo, *params, **kw):
         bh64_key = (
             key.derive_key(salt=b"", domain=b"buzhash64", size=32, from_id_key=True) if key is not None else b"\0" * 32
         )
-        return ChunkerBuzHash64(bh64_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse)
+        return _made(algo, ChunkerBuzHash64(bh64_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse))
     if algo == "fastcdc":
         # keyed gear table, derived from the id key (own domain). params is
         # (chunk_min_exp, chunk_max_exp, hash_mask_bits, nc_level) - no window (Gear is window-less).
         fc_key = (
             key.derive_key(salt=b"", domain=b"fastcdc", size=32, from_id_key=True) if key is not None else b"\0" * 32
         )
-        return ChunkerFastCDC(fc_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse)
+        return _made(algo, ChunkerFastCDC(fc_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse))
     if algo == "rabin-aes":
         # UHF-then-PRF chunker (secret Rabin polynomial + AES-128), derived from
         # the id key (own domain). params is (chunk_min_exp, chunk_max_exp,
@@ -38,7 +63,7 @@ def get_chunker(algo, *params, **kw):
         ra_key = (
             key.derive_key(salt=b"", domain=b"rabin-aes", size=32, from_id_key=True) if key is not None else b"\0" * 32
         )
-        return ChunkerRabinAES(ra_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse)
+        return _made(algo, ChunkerRabinAES(ra_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse))
     if algo == "goldilocks-aes":
         # like rabin-aes, but with the Goldilocks prime-field polynomial hash as
         # the UHF (the reference construction of eprint 2025/558). Same param
@@ -48,7 +73,7 @@ def get_chunker(algo, *params, **kw):
             if key is not None
             else b"\0" * 32
         )
-        return ChunkerGoldilocksAES(gl_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse)
+        return _made(algo, ChunkerGoldilocksAES(gl_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse))
     if algo == "toeplitz-aes":
         # like rabin-aes, but with a tabulated LFSR/Toeplitz hash as the UHF
         # (secret 2 KiB table, fixed public polynomial). Same param shape:
@@ -58,9 +83,9 @@ def get_chunker(algo, *params, **kw):
             if key is not None
             else b"\0" * 32
         )
-        return ChunkerToeplitzAES(tp_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse)
+        return _made(algo, ChunkerToeplitzAES(tp_key, *params, normal_size=kw.get("normal_size", 0), sparse=sparse))
     if algo == "fixed":
-        return ChunkerFixed(*params, sparse=sparse)
+        return _made(algo, ChunkerFixed(*params, sparse=sparse))
     if algo == "fail":
-        return ChunkerFailing(*params)
+        return _made(algo, ChunkerFailing(*params))
     raise TypeError("unsupported chunker algo %r" % algo)

From be024a5b771602f81186971fef7d6621d2935eed Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 22:22:18 +0200
Subject: [PATCH 046/144] benchmark cpu: add a throughput column, #10049

The output only gave the time for a fixed amount of data, so comparing
rows meant dividing in your head. Every row now ends in MB/s - always
MB/s, never scaled to kB/s or GB/s, so the numbers stay comparable
between a chunker at 3500 and lzma at 5.

    fastcdc,19,23,21,2         1.00 GB    0.283s    3537.6 MB/s
    lzma,6                   100.00 MB   18.815s       5.3 MB/s

msgpack packs items rather than bytes, so it reports kItems/s instead;
that row already had "100k Items" where the others have a byte count.

The reported sizes are now derived from the work actually done
(data_size * iterations) instead of being hardcoded. In the normal run
that is the same 1 GB and 100 MB as before, but under
_BORG_BENCHMARK_CPU_TEST - one iteration over 100 kB, used to keep CI
fast - the old constants claimed 1 GB for 100 kB of work, which would
have made the new column wrong by four orders of magnitude. The msgpack
item count had the same problem and is now counted the same way.

JSON output is unchanged apart from those honest sizes: throughput is
size/time, so a consumer can compute it and does not need a new field.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                   |  1 +
 src/borg/archiver/benchmark_cmd.py | 28 +++++++++++++++++-----------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index a76c3ccb4b..57f874da64 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -200,6 +200,7 @@ New features:
 - analyze: report deduplicated size of a set of archives, #5741
 - repo-compress: was temporarily gone, re-added now with pack support, #9663
 - version: add --json output, #10004
+- benchmark cpu: add a throughput column (MB/s), #10049
 - completion: generate fish completions, remove hand-written ones, #9989
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index b1a81abadf..3d826e6da6 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -153,6 +153,10 @@ def do_benchmark_cpu(self, args):
         """Benchmark CPU-bound operations."""
         from timeit import timeit
 
+        def throughput(size, dt):
+            """Rate as MB/s, always - so numbers stay comparable between rows."""
+            return f"{size / dt / 1e6:>8.1f} MB/s"
+
         result = {} if args.json else None
 
         is_test = "_BORG_BENCHMARK_CPU_TEST" in os.environ
@@ -173,7 +177,7 @@ def do_benchmark_cpu(self, args):
             print("Chunkers =======================================================")
         else:
             result["chunkers"] = []
-        size = 1000000000
+        size = data_size * number_default
 
         def chunkit(ch):
             with io.BytesIO(random_10M) as data_file:
@@ -224,7 +228,7 @@ def chunkit(ch):
                 algo, _, algo_params = spec.partition(",")
                 result["chunkers"].append({"algo": algo, "algo_params": algo_params, "size": size, "time": dt})
             else:
-                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s")
+                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
 
         from ..crypto.low_level import hmac_sha256, blake2b_256
         import blake3
@@ -233,7 +237,7 @@ def chunkit(ch):
             print("Cryptographic hashes / MACs ====================================")
         else:
             result["hashes"] = []
-        size = 1000000000
+        size = data_size * number_default
         hashes_tests = [
             ("hmac-sha256", lambda: hmac_sha256(key_256, random_10M)),
             ("blake2b-256", lambda: blake2b_256(key_256, random_10M)),
@@ -245,7 +249,7 @@ def chunkit(ch):
             if args.json:
                 result["hashes"].append({"algo": spec, "size": size, "time": dt})
             else:
-                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s")
+                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
 
         from ..crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
         from ..crypto.low_level import AES256_OCB, CHACHA20_POLY1305
@@ -254,7 +258,7 @@ def chunkit(ch):
             print("Encryption =====================================================")
         else:
             result["encryption"] = []
-        size = 1000000000
+        size = data_size * number_default
 
         tests = [
             (
@@ -285,7 +289,7 @@ def chunkit(ch):
             if args.json:
                 result["encryption"].append({"algo": spec, "size": size, "time": dt})
             else:
-                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s")
+                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
 
         if not args.json:
             print("Compression ====================================================")
@@ -307,13 +311,13 @@ def chunkit(ch):
             "lzma,9",
         ]:
             compressor = CompressionSpec(spec).compressor
-            size = 100000000
+            size = data_size * number_compression
             dt = timeit(lambda: compressor.compress({}, random_10M), number=number_compression)
             if args.json:
                 algo, _, algo_params = spec.partition(",")
                 result["compression"].append({"algo": algo, "algo_params": algo_params, "size": size, "time": dt})
             else:
-                print(f"{spec:<12} {format_file_size(size):<10} {dt:.3f}s")
+                print(f"{spec:<12} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
 
         if not args.json:
             print("msgpack ========================================================")
@@ -321,13 +325,15 @@ def chunkit(ch):
             result["msgpack"] = []
         item = Item(path="foo/bar/baz", mode=660, mtime=1234567)
         items = [item.as_dict()] * 1000
-        size = "100k Items"
+        count = 1000 * number_default
+        size = "%dk Items" % (count // 1000)
         spec = "msgpack"
         dt = timeit(lambda: msgpack.packb(items), number=number_default)
         if args.json:
-            result["msgpack"].append({"algo": spec, "count": 100000, "time": dt})
+            result["msgpack"].append({"algo": spec, "count": count, "time": dt})
         else:
-            print(f"{spec:<12} {size:<10} {dt:.3f}s")
+            # this one packs items, not bytes, so it gets a rate in its own unit
+            print(f"{spec:<12} {size:<10} {dt:.3f}s  {count / dt / 1000:>8.1f} kItems/s")
 
         if args.json:
             json_print(result)

From af3c1729601ef3e785ce8457cfcd237f2d647810 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 22:22:28 +0200
Subject: [PATCH 047/144] benchmark cpu: buffer sizes that matter, and section
 selection, #10050

blake3 and zstd only use multiple threads above a size threshold, so a
single buffer size hid most of what there is to see. The hashes are now
measured at 100kiB (below the threshold), 2MiB (a typical borg chunk) and
50MiB (a borg pack), the compressors at 100kiB and 2MiB. Every row still
processes about the same total number of bytes, so the throughput column
stays comparable across sizes.

That difference is the whole point of the change:

    blake3-mt (100kiB)         999.94 MB  0.851s    1175.3 MB/s
    blake3-mt (2MiB)           998.24 MB  0.078s   12867.8 MB/s
    blake3-mt (50MiB)          996.15 MB  0.058s   17294.7 MB/s

Algorithms whose behaviour does not depend on the buffer size get one row
at 2MiB rather than three identical ones.

Adds --chunking, --hashing, --encryption, --compression and --msgpack;
giving none runs everything, as before. The issue asked for the first
four - msgpack got one too, so that every section can be selected and
none is stuck always running.

The full run does not get slower despite having 46 rows instead of 28:
the compressors now do half as many bytes per size, so their total is
unchanged, and they dominate the runtime.

JSON entries for hashes and compression gain a buffer_size field, without
which rows for the same algorithm could no longer be told apart.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                              |   3 +
 src/borg/archiver/benchmark_cmd.py            | 378 ++++++++++--------
 .../testsuite/archiver/benchmark_cmd_test.py  |  23 ++
 3 files changed, 243 insertions(+), 161 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 57f874da64..15e5d31209 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -201,6 +201,9 @@ New features:
 - repo-compress: was temporarily gone, re-added now with pack support, #9663
 - version: add --json output, #10004
 - benchmark cpu: add a throughput column (MB/s), #10049
+- benchmark cpu: measure hashes and compressors at several buffer sizes, and
+  add --chunking / --hashing / --encryption / --compression / --msgpack to run
+  only a subset, #10050
 - completion: generate fish completions, remove hand-written ones, #9989
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index 3d826e6da6..f8c207c337 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -153,187 +153,229 @@ def do_benchmark_cpu(self, args):
         """Benchmark CPU-bound operations."""
         from timeit import timeit
 
-        def throughput(size, dt):
-            """Rate as MB/s, always - so numbers stay comparable between rows."""
-            return f"{size / dt / 1e6:>8.1f} MB/s"
-
         result = {} if args.json else None
 
+        # no section selected means: run them all
+        sections = ("chunking", "hashing", "encryption", "compression", "msgpack")
+        selected = {name for name in sections if getattr(args, name)}
+        if not selected:
+            selected = set(sections)
+
         is_test = "_BORG_BENCHMARK_CPU_TEST" in os.environ
-        # Use minimal iterations and data size in test mode to keep CI fast.
+        # Use minimal data and one iteration in test mode to keep CI fast.
         number_default = 1 if is_test else 100
-        number_compression = 1 if is_test else 10
-        data_size = 100 * 1000 if is_test else 10 * 1000 * 1000
+        chunk_data_size = 100 * 1000 if is_test else 10 * 1000 * 1000
+
+        # Buffer sizes matter where an algorithm switches to multiple threads:
+        # blake3 and zstd do so above a threshold, so a size below it, one the
+        # size of a typical borg chunk and one the size of a borg pack behave
+        # quite differently. Each row processes about as many bytes in total,
+        # so the throughput column stays comparable across sizes.
+        SMALL, CHUNK, PACK = ("100kiB", 100 * 1024), ("2MiB", 2 * 1024 * 1024), ("50MiB", 50 * 1024 * 1024)
+        if is_test:
+            hash_sizes = comp_sizes = one_size = [SMALL]
+            hash_total = comp_total = SMALL[1]
+        else:
+            hash_sizes = [SMALL, CHUNK, PACK]
+            comp_sizes = [SMALL, CHUNK]
+            one_size = [CHUNK]  # for algorithms where the buffer size does not change behaviour
+            hash_total = 1000 * 1000 * 1000
+            comp_total = 50 * 1000 * 1000
 
-        random_10M = os.urandom(data_size)
-        key_256 = os.urandom(32)
-        key_128 = os.urandom(16)
-        key_96 = os.urandom(12)
+        buffers = {}
 
-        import io
-        from ..chunkers import get_chunker, release_chunk_data  # noqa
+        def buffer(nbytes):
+            """Random data of the given size, made once and reused."""
+            if nbytes not in buffers:
+                buffers[nbytes] = os.urandom(nbytes)
+            return buffers[nbytes]
 
-        if not args.json:
-            print("Chunkers =======================================================")
-        else:
-            result["chunkers"] = []
-        size = data_size * number_default
-
-        def chunkit(ch):
-            with io.BytesIO(random_10M) as data_file:
-                for chunk in ch.chunkify(fd=data_file):
-                    release_chunk_data(chunk.data)
-
-        for spec, setup, func, vars in [
-            ("fixed,1048576", "ch = get_chunker('fixed', 1048576, sparse=False)", "chunkit(ch)", locals()),
-            # fastcdc (window-less keyed gear hash); gear table creation is slow, keep it in setup
-            ("fastcdc,19,23,21,2", "ch = get_chunker('fastcdc', 19, 23, 21, 2, sparse=False)", "chunkit(ch)", locals()),
-            # note: the buzhash64 chunker creation is rather slow, so we must keep it in setup
-            (
-                "buzhash64,19,23,21,4095,2",
-                "ch = get_chunker('buzhash64', 19, 23, 21, 4095, 2, sparse=False)",
-                "chunkit(ch)",
-                locals(),
-            ),
-            (
-                "buzhash,19,23,21,4095",
-                "ch = get_chunker('buzhash', 19, 23, 21, 4095, sparse=False)",
-                "chunkit(ch)",
-                locals(),
-            ),
-            # toeplitz-aes (UHF-then-PRF, tabulated Toeplitz hash); table creation in setup
-            (
-                "toeplitz-aes,19,23,21,2",
-                "ch = get_chunker('toeplitz-aes', 19, 23, 21, 2, sparse=False)",
-                "chunkit(ch)",
-                locals(),
-            ),
-            # rabin-aes (UHF-then-PRF); polynomial sampling / table creation is slow, keep it in setup
-            (
-                "rabin-aes,19,23,21,2",
-                "ch = get_chunker('rabin-aes', 19, 23, 21, 2, sparse=False)",
-                "chunkit(ch)",
-                locals(),
-            ),
-            # goldilocks-aes (UHF-then-PRF, prime-field comparison chunker); table creation in setup
-            (
-                "goldilocks-aes,19,23,21,2",
-                "ch = get_chunker('goldilocks-aes', 19, 23, 21, 2, sparse=False)",
-                "chunkit(ch)",
-                locals(),
-            ),
-        ]:
-            dt = timeit(func, setup, number=number_default, globals=vars)
+        def throughput(size, dt):
+            """Rate as MB/s, always - so numbers stay comparable between rows."""
+            return f"{size / dt / 1e6:>8.1f} MB/s"
+
+        def report(section, spec, size, dt, width, **extra):
             if args.json:
-                algo, _, algo_params = spec.partition(",")
-                result["chunkers"].append({"algo": algo, "algo_params": algo_params, "size": size, "time": dt})
+                result[section].append({"size": size, "time": dt, **extra})
             else:
-                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
-
-        from ..crypto.low_level import hmac_sha256, blake2b_256
-        import blake3
+                print(f"{spec:<{width}} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
 
-        if not args.json:
-            print("Cryptographic hashes / MACs ====================================")
-        else:
-            result["hashes"] = []
-        size = data_size * number_default
-        hashes_tests = [
-            ("hmac-sha256", lambda: hmac_sha256(key_256, random_10M)),
-            ("blake2b-256", lambda: blake2b_256(key_256, random_10M)),
-            ("blake3", lambda: blake3.blake3(random_10M, key=key_256).digest()),
-            ("blake3-mt", lambda: blake3.blake3(random_10M, key=key_256, max_threads=blake3.blake3.AUTO).digest()),
-        ]
-        for spec, func in hashes_tests:
-            dt = timeit(func, number=number_default)
+        def section_header(section, title):
             if args.json:
-                result["hashes"].append({"algo": spec, "size": size, "time": dt})
+                result[section] = []
             else:
-                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
+                print(f"{title} ".ljust(64, "="))
 
-        from ..crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
-        from ..crypto.low_level import AES256_OCB, CHACHA20_POLY1305
+        random_10M = buffer(chunk_data_size)
+        key_256 = os.urandom(32)
+        key_128 = os.urandom(16)
+        key_96 = os.urandom(12)
 
-        if not args.json:
-            print("Encryption =====================================================")
-        else:
-            result["encryption"] = []
-        size = data_size * number_default
-
-        tests = [
-            (
-                "aes-256-ctr-hmac-sha256",
-                lambda: AES256_CTR_HMAC_SHA256(key_256, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
-                    random_10M, header=b"X"
+        if "chunking" in selected:
+            import io
+            from ..chunkers import get_chunker, release_chunk_data  # noqa
+
+            section_header("chunkers", "Chunkers")
+            size = chunk_data_size * number_default
+
+            def chunkit(ch):
+                with io.BytesIO(random_10M) as data_file:
+                    for chunk in ch.chunkify(fd=data_file):
+                        release_chunk_data(chunk.data)
+
+            for spec, setup, func, vars in [
+                ("fixed,1048576", "ch = get_chunker('fixed', 1048576, sparse=False)", "chunkit(ch)", locals()),
+                # fastcdc (window-less keyed gear hash); gear table creation is slow, keep it in setup
+                (
+                    "fastcdc,19,23,21,2",
+                    "ch = get_chunker('fastcdc', 19, 23, 21, 2, sparse=False)",
+                    "chunkit(ch)",
+                    locals(),
                 ),
-            ),
-            (
-                "aes-256-ctr-blake2b",
-                lambda: AES256_CTR_BLAKE2b(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
-                    random_10M, header=b"X"
+                # note: the buzhash64 chunker creation is rather slow, so we must keep it in setup
+                (
+                    "buzhash64,19,23,21,4095,2",
+                    "ch = get_chunker('buzhash64', 19, 23, 21, 4095, 2, sparse=False)",
+                    "chunkit(ch)",
+                    locals(),
                 ),
-            ),
-            (
-                "aes-256-ocb",
-                lambda: AES256_OCB(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(random_10M, header=b"X"),
-            ),
-            (
-                "chacha20-poly1305",
-                lambda: CHACHA20_POLY1305(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(
-                    random_10M, header=b"X"
+                (
+                    "buzhash,19,23,21,4095",
+                    "ch = get_chunker('buzhash', 19, 23, 21, 4095, sparse=False)",
+                    "chunkit(ch)",
+                    locals(),
                 ),
-            ),
-        ]
-        for spec, func in tests:
-            dt = timeit(func, number=number_default)
-            if args.json:
-                result["encryption"].append({"algo": spec, "size": size, "time": dt})
-            else:
-                print(f"{spec:<26} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
+                # toeplitz-aes (UHF-then-PRF, tabulated Toeplitz hash); table creation in setup
+                (
+                    "toeplitz-aes,19,23,21,2",
+                    "ch = get_chunker('toeplitz-aes', 19, 23, 21, 2, sparse=False)",
+                    "chunkit(ch)",
+                    locals(),
+                ),
+                # rabin-aes (UHF-then-PRF); polynomial sampling / table creation is slow, keep it in setup
+                (
+                    "rabin-aes,19,23,21,2",
+                    "ch = get_chunker('rabin-aes', 19, 23, 21, 2, sparse=False)",
+                    "chunkit(ch)",
+                    locals(),
+                ),
+                # goldilocks-aes (UHF-then-PRF, prime-field comparison chunker); table creation in setup
+                (
+                    "goldilocks-aes,19,23,21,2",
+                    "ch = get_chunker('goldilocks-aes', 19, 23, 21, 2, sparse=False)",
+                    "chunkit(ch)",
+                    locals(),
+                ),
+            ]:
+                dt = timeit(func, setup, number=number_default, globals=vars)
+                algo, _, algo_params = spec.partition(",")
+                report("chunkers", spec, size, dt, 26, algo=algo, algo_params=algo_params)
+
+        if "hashing" in selected:
+            from ..crypto.low_level import hmac_sha256, blake2b_256
+            import blake3
+
+            section_header("hashes", "Cryptographic hashes / MACs")
+            # only blake3 uses multiple threads, and only above a size threshold,
+            # so it is the one worth measuring at several buffer sizes.
+            hashes_tests = [
+                ("hmac-sha256", one_size, lambda d: hmac_sha256(key_256, d)),
+                ("blake2b-256", one_size, lambda d: blake2b_256(key_256, d)),
+                ("blake3", hash_sizes, lambda d: blake3.blake3(d, key=key_256).digest()),
+                (
+                    "blake3-mt",
+                    hash_sizes,
+                    lambda d: blake3.blake3(d, key=key_256, max_threads=blake3.blake3.AUTO).digest(),
+                ),
+            ]
+            for spec, sizes, func in hashes_tests:
+                for label, nbytes in sizes:
+                    data = buffer(nbytes)
+                    number = max(1, hash_total // nbytes)
+                    dt = timeit(lambda: func(data), number=number)
+                    report("hashes", f"{spec} ({label})", nbytes * number, dt, 26, algo=spec, buffer_size=nbytes)
+
+        if "encryption" in selected:
+            from ..crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
+            from ..crypto.low_level import AES256_OCB, CHACHA20_POLY1305
+
+            section_header("encryption", "Encryption")
+            size = chunk_data_size * number_default
+            tests = [
+                (
+                    "aes-256-ctr-hmac-sha256",
+                    lambda: AES256_CTR_HMAC_SHA256(key_256, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
+                        random_10M, header=b"X"
+                    ),
+                ),
+                (
+                    "aes-256-ctr-blake2b",
+                    lambda: AES256_CTR_BLAKE2b(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
+                        random_10M, header=b"X"
+                    ),
+                ),
+                (
+                    "aes-256-ocb",
+                    lambda: AES256_OCB(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(random_10M, header=b"X"),
+                ),
+                (
+                    "chacha20-poly1305",
+                    lambda: CHACHA20_POLY1305(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(
+                        random_10M, header=b"X"
+                    ),
+                ),
+            ]
+            for spec, func in tests:
+                dt = timeit(func, number=number_default)
+                report("encryption", spec, size, dt, 26, algo=spec)
+
+        if "compression" in selected:
+            section_header("compression", "Compression")
+            for spec in [
+                "lz4",
+                "zstd,1",
+                "zstd,3",
+                "zstd,5",
+                "zstd,10",
+                "zstd,16",
+                "zstd,22",
+                "zlib,0",
+                "zlib,6",
+                "zlib,9",
+                "lzma,0",
+                "lzma,6",
+                "lzma,9",
+            ]:
+                compressor = CompressionSpec(spec).compressor
+                for label, nbytes in comp_sizes:
+                    data = buffer(nbytes)
+                    number = max(1, comp_total // nbytes)
+                    dt = timeit(lambda: compressor.compress({}, data), number=number)
+                    algo, _, algo_params = spec.partition(",")
+                    report(
+                        "compression",
+                        f"{spec} ({label})",
+                        nbytes * number,
+                        dt,
+                        20,
+                        algo=algo,
+                        algo_params=algo_params,
+                        buffer_size=nbytes,
+                    )
 
-        if not args.json:
-            print("Compression ====================================================")
-        else:
-            result["compression"] = []
-        for spec in [
-            "lz4",
-            "zstd,1",
-            "zstd,3",
-            "zstd,5",
-            "zstd,10",
-            "zstd,16",
-            "zstd,22",
-            "zlib,0",
-            "zlib,6",
-            "zlib,9",
-            "lzma,0",
-            "lzma,6",
-            "lzma,9",
-        ]:
-            compressor = CompressionSpec(spec).compressor
-            size = data_size * number_compression
-            dt = timeit(lambda: compressor.compress({}, random_10M), number=number_compression)
+        if "msgpack" in selected:
+            section_header("msgpack", "msgpack")
+            item = Item(path="foo/bar/baz", mode=660, mtime=1234567)
+            items = [item.as_dict()] * 1000
+            count = 1000 * number_default
+            spec = "msgpack"
+            dt = timeit(lambda: msgpack.packb(items), number=number_default)
             if args.json:
-                algo, _, algo_params = spec.partition(",")
-                result["compression"].append({"algo": algo, "algo_params": algo_params, "size": size, "time": dt})
+                result["msgpack"].append({"algo": spec, "count": count, "time": dt})
             else:
-                print(f"{spec:<12} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
-
-        if not args.json:
-            print("msgpack ========================================================")
-        else:
-            result["msgpack"] = []
-        item = Item(path="foo/bar/baz", mode=660, mtime=1234567)
-        items = [item.as_dict()] * 1000
-        count = 1000 * number_default
-        size = "%dk Items" % (count // 1000)
-        spec = "msgpack"
-        dt = timeit(lambda: msgpack.packb(items), number=number_default)
-        if args.json:
-            result["msgpack"].append({"algo": spec, "count": count, "time": dt})
-        else:
-            # this one packs items, not bytes, so it gets a rate in its own unit
-            print(f"{spec:<12} {size:<10} {dt:.3f}s  {count / dt / 1000:>8.1f} kItems/s")
+                # this one packs items, not bytes, so it gets a rate in its own unit
+                size = "%dk Items" % (count // 1000)
+                print(f"{spec:<20} {size:<10} {dt:.3f}s  {count / dt / 1000:>8.1f} kItems/s")
 
         if args.json:
             json_print(result)
@@ -412,6 +454,15 @@ def build_parser_benchmarks(self, subparsers, common_parser, mid_common_parser):
 
         - an otherwise as idle as possible machine
         - enough free memory so there will be no slow down due to paging activity
+
+        By default all benchmarks run. Give one or more of --chunking, --hashing,
+        --encryption, --compression, --msgpack to run only those.
+
+        Some algorithms use multiple threads only above a size threshold, so the
+        hashes and compressors are measured at several buffer sizes: 100kiB (below
+        the threshold), 2MiB (a typical borg chunk) and, for hashes, 50MiB (a borg
+        pack). Every row processes roughly the same total number of bytes, so the
+        throughput column is comparable between them.
         """
         )
         subparser = ArgumentParser(
@@ -419,3 +470,8 @@ def build_parser_benchmarks(self, subparsers, common_parser, mid_common_parser):
         )
         benchmark_parsers.add_subcommand("cpu", subparser, help="benchmarks Borg CPU-bound operations.")
         subparser.add_argument("--json", action="store_true", help="format output as JSON")
+        subparser.add_argument("--chunking", action="store_true", help="benchmark the chunkers")
+        subparser.add_argument("--hashing", action="store_true", help="benchmark the hashes / MACs")
+        subparser.add_argument("--encryption", action="store_true", help="benchmark the encryption modes")
+        subparser.add_argument("--compression", action="store_true", help="benchmark the compressors")
+        subparser.add_argument("--msgpack", action="store_true", help="benchmark msgpack item packing")
diff --git a/src/borg/testsuite/archiver/benchmark_cmd_test.py b/src/borg/testsuite/archiver/benchmark_cmd_test.py
index 5e0375fa2a..e04021ab02 100644
--- a/src/borg/testsuite/archiver/benchmark_cmd_test.py
+++ b/src/borg/testsuite/archiver/benchmark_cmd_test.py
@@ -1,5 +1,7 @@
 import json
 
+import pytest
+
 from ...constants import *  # NOQA
 from . import cmd, RK_ENCRYPTION
 
@@ -56,6 +58,27 @@ def test_benchmark_cpu(archiver, monkeypatch):
     assert "msgpack" in output
 
 
+@pytest.mark.parametrize(
+    "flag, header",
+    [
+        ("--chunking", "Chunkers"),
+        ("--hashing", "Cryptographic hashes / MACs"),
+        ("--encryption", "Encryption"),
+        ("--compression", "Compression"),
+        ("--msgpack", "msgpack"),
+    ],
+)
+def test_benchmark_cpu_section_selection(archiver, monkeypatch, flag, header):
+    # a flag runs that section and only that one
+    monkeypatch.setenv("_BORG_BENCHMARK_CPU_TEST", "YES")
+    all_headers = ["Chunkers", "Cryptographic hashes / MACs", "Encryption", "Compression", "msgpack"]
+    output = cmd(archiver, "benchmark", "cpu", flag)
+    assert header in output
+    for other in all_headers:
+        if other != header:
+            assert other not in output
+
+
 def test_benchmark_cpu_json(archiver, monkeypatch):
     monkeypatch.setenv("_BORG_BENCHMARK_CPU_TEST", "YES")
     output = cmd(archiver, "benchmark", "cpu", "--json")

From 7e11a20eefc5e9de2eb5da22359be016d761601d Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 22:22:31 +0200
Subject: [PATCH 048/144] benchmark cpu: compress compressible data, not random
 noise

The compression section fed os.urandom() to every codec. Nothing
compresses, so they all took their incompressible fast path: the levels
barely differed, and zstd's multi-threading looked like a 2-5x loss when
on real data it is a 1.7-3x win. The section measured a case no backup
ever hits.

It now compresses a deterministic word soup instead - built from a fixed
seed with a hand-written xorshift, so the bytes do not depend on the
Python version's PRNG internals and every machine and run sees the same
input. It compresses about 4.3x at zstd,3, and the section prints that
ratio so the numbers can be read.

The multi-threading threshold now shows up with the right sign:

    zstd,1 (100kiB)      9.93 MB    0.018s     562.7 MB/s
    zstd,1 (2MiB)        8.39 MB    0.006s    1417.6 MB/s

Only the compressors change. The chunkers keep random data, which they
need for realistic cut points, and hash/encryption throughput does not
depend on the content.

Bytes per row drop from 50 to 10 MB: with real work to do the slow codecs
are much slower, and the section went from ~30 to 91 seconds. At 10 MB it
takes 18, with a floor of 3 repetitions so the fast codecs still get a
few.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                   |  1 +
 src/borg/archiver/benchmark_cmd.py | 45 ++++++++++++++++++++++++++++--
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 15e5d31209..052c4d186b 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -204,6 +204,7 @@ New features:
 - benchmark cpu: measure hashes and compressors at several buffer sizes, and
   add --chunking / --hashing / --encryption / --compression / --msgpack to run
   only a subset, #10050
+- benchmark cpu: compress deterministic compressible data instead of random noise
 - completion: generate fish completions, remove hand-written ones, #9989
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index f8c207c337..0a9cbafe3d 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -180,7 +180,9 @@ def do_benchmark_cpu(self, args):
             comp_sizes = [SMALL, CHUNK]
             one_size = [CHUNK]  # for algorithms where the buffer size does not change behaviour
             hash_total = 1000 * 1000 * 1000
-            comp_total = 50 * 1000 * 1000
+            # compressible data means the codecs actually work, so the slow ones
+            # (lzma, high zstd levels) need fewer bytes to stay bearable
+            comp_total = 10 * 1000 * 1000
 
         buffers = {}
 
@@ -190,6 +192,39 @@ def buffer(nbytes):
                 buffers[nbytes] = os.urandom(nbytes)
             return buffers[nbytes]
 
+        compressible = {}
+
+        def compressible_buffer(nbytes):
+            """Deterministic, text-like, compressible data of the given size.
+
+            Random bytes are the worst possible input for a compression benchmark:
+            nothing compresses, so every codec takes its incompressible fast path,
+            the levels barely differ, and zstd's multi-threading looks like a large
+            loss when on real data it is a 1.7 .. 3x win. This is a word soup built
+            from a fixed seed - identical on every machine and every run, so numbers
+            stay comparable - which compresses about 4x at zstd,3.
+
+            The xorshift is written out rather than using random.Random so that the
+            bytes do not depend on the Python version's PRNG internals.
+            """
+            if nbytes not in compressible:
+                words = (
+                    b"backup archive chunk repository compress encrypt index cache manifest "
+                    b"the and of a to in is it for with data borg file path size key id hash "
+                    b"item repo object segment directory user group mtime mode owner content"
+                ).split()
+                out = bytearray()
+                state = 0x2545F4914F6CDD1D
+                mask = 0xFFFFFFFFFFFFFFFF
+                while len(out) < nbytes:
+                    state ^= (state << 13) & mask
+                    state ^= state >> 7
+                    state ^= (state << 17) & mask
+                    out += words[state % len(words)]
+                    out += b"\n" if state & 0xF == 0 else b" "
+                compressible[nbytes] = bytes(out[:nbytes])
+            return compressible[nbytes]
+
         def throughput(size, dt):
             """Rate as MB/s, always - so numbers stay comparable between rows."""
             return f"{size / dt / 1e6:>8.1f} MB/s"
@@ -331,6 +366,10 @@ def chunkit(ch):
 
         if "compression" in selected:
             section_header("compression", "Compression")
+            if not args.json:
+                sample = compressible_buffer(comp_sizes[-1][1])
+                ratio = len(sample) / len(CompressionSpec("zstd,3").compressor.compress({}, sample)[1])
+                print(f"(test data is compressible, {ratio:.1f}x at zstd,3)")
             for spec in [
                 "lz4",
                 "zstd,1",
@@ -348,8 +387,8 @@ def chunkit(ch):
             ]:
                 compressor = CompressionSpec(spec).compressor
                 for label, nbytes in comp_sizes:
-                    data = buffer(nbytes)
-                    number = max(1, comp_total // nbytes)
+                    data = compressible_buffer(nbytes)
+                    number = max(3, comp_total // nbytes)  # a few reps even for the fast codecs
                     dt = timeit(lambda: compressor.compress({}, data), number=number)
                     algo, _, algo_params = spec.partition(",")
                     report(

From bf99d5a2525d2033a08752f816e62415fd003d81 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 22:22:34 +0200
Subject: [PATCH 049/144] benchmark cpu: measure blake3 the way borg uses it

There were two rows, blake3 and blake3-mt, but borg has no such choice:
AESKeyBase.id_hash passes max_threads=AUTO only when the chunk reaches
get_blake3_mt_threshold() (256 KiB by default), and 1 below it. So one of
the two rows was always measuring something borg would not do at that
size.

Now there is a single blake3 row applying that same rule, so each buffer
size shows what borg would really do with a chunk of that size:

    blake3 (100kiB)            999.94 MB  0.442s    2260.1 MB/s
    blake3 (2MiB)              998.24 MB  0.077s   12968.2 MB/s
    blake3 (50MiB)             996.15 MB  0.057s   17623.1 MB/s

The 100kiB row is single-threaded and the faster for it - which is the
threshold doing its job, not a penalty.

It calls get_blake3_mt_threshold() rather than hardcoding the size, so
BORG_BLAKE3_MT_THRESHOLD moves the benchmark and borg together and the
benchmark can be used to tune it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                   |  2 ++
 src/borg/archiver/benchmark_cmd.py | 14 ++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 052c4d186b..0dc7d96874 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -205,6 +205,8 @@ New features:
   add --chunking / --hashing / --encryption / --compression / --msgpack to run
   only a subset, #10050
 - benchmark cpu: compress deterministic compressible data instead of random noise
+- benchmark cpu: measure blake3 the way borg uses it (multi-threaded only above
+  BORG_BLAKE3_MT_THRESHOLD)
 - completion: generate fish completions, remove hand-written ones, #9989
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index 0a9cbafe3d..e7c555467d 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -308,20 +308,22 @@ def chunkit(ch):
 
         if "hashing" in selected:
             from ..crypto.low_level import hmac_sha256, blake2b_256
+            from ..crypto.key import get_blake3_mt_threshold
             import blake3
 
+            def blake3_hash(d):
+                # mirror what borg does per chunk (see AESKeyBase.id_hash): below the
+                # threshold multi-threading costs more than it gains, so it is not used
+                max_threads = blake3.blake3.AUTO if len(d) >= get_blake3_mt_threshold() else 1
+                return blake3.blake3(d, key=key_256, max_threads=max_threads).digest()
+
             section_header("hashes", "Cryptographic hashes / MACs")
             # only blake3 uses multiple threads, and only above a size threshold,
             # so it is the one worth measuring at several buffer sizes.
             hashes_tests = [
                 ("hmac-sha256", one_size, lambda d: hmac_sha256(key_256, d)),
                 ("blake2b-256", one_size, lambda d: blake2b_256(key_256, d)),
-                ("blake3", hash_sizes, lambda d: blake3.blake3(d, key=key_256).digest()),
-                (
-                    "blake3-mt",
-                    hash_sizes,
-                    lambda d: blake3.blake3(d, key=key_256, max_threads=blake3.blake3.AUTO).digest(),
-                ),
+                ("blake3", hash_sizes, blake3_hash),
             ]
             for spec, sizes, func in hashes_tests:
                 for label, nbytes in sizes:

From 62127a5d35dac0d02abb44199142a3ea2aa1a491 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 22:49:25 +0200
Subject: [PATCH 050/144] benchmark cpu: use binary data sizes, 1 GiB and 10
 MiB

The data volumes were decimal (10^9 and 10^7 bytes) and the buffer sizes
binary, so the totals landed on awkward values and the size column read
"998.24 MB" or "9.93 MB" depending on which buffer a row used.

Every row now processes exactly 1 GiB, or 10 MiB for the compressors,
and says so:

    blake3 (128kiB)            1.00 GiB    0.474s    2266.5 MB/s
    blake3 (2MiB)              1.00 GiB    0.085s   12590.2 MB/s
    lz4 (128kiB)               10.00 MiB   0.011s     988.4 MB/s

Exactness needs buffers that divide those totals, so the two that did not
became powers of two: 100kiB -> 128kiB and 50MiB -> 64MiB for the hashes,
and the chunker/encryption buffer 10 MB -> 8 MiB. They serve the same
purpose as before - 128kiB is still below blake3's 256 KiB threshold and
zstd's 768 KiB one, 64MiB is still pack sized.

The size column now prints IEC units directly rather than going through
format_file_size. These volumes are constants of the benchmark, not user
data: following BORG_UNITS would make two people's output incomparable,
and would render 1 GiB as "1.07 GB".

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/archiver/benchmark_cmd.py | 31 ++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index e7c555467d..31da9a7fe8 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -7,6 +7,7 @@
 
 from ..constants import *  # NOQA
 from ..helpers import format_file_size, CompressionSpec
+from ..helpers import sizeof_fmt_iec
 from ..helpers import FilesystemDirSpec
 from ..helpers import json_print
 from ..helpers import msgpack
@@ -161,17 +162,21 @@ def do_benchmark_cpu(self, args):
         if not selected:
             selected = set(sections)
 
+        KIB, MIB, GIB = 1024, 1024 * 1024, 1024 * 1024 * 1024
+
         is_test = "_BORG_BENCHMARK_CPU_TEST" in os.environ
         # Use minimal data and one iteration in test mode to keep CI fast.
-        number_default = 1 if is_test else 100
-        chunk_data_size = 100 * 1000 if is_test else 10 * 1000 * 1000
+        chunk_data_size = 128 * KIB if is_test else 8 * MIB
+        number_default = 1 if is_test else GIB // chunk_data_size
 
         # Buffer sizes matter where an algorithm switches to multiple threads:
         # blake3 and zstd do so above a threshold, so a size below it, one the
         # size of a typical borg chunk and one the size of a borg pack behave
-        # quite differently. Each row processes about as many bytes in total,
-        # so the throughput column stays comparable across sizes.
-        SMALL, CHUNK, PACK = ("100kiB", 100 * 1024), ("2MiB", 2 * 1024 * 1024), ("50MiB", 50 * 1024 * 1024)
+        # quite differently. Every row processes the same total either way, so
+        # the throughput column stays comparable across sizes.
+        # powers of two, so every row processes exactly 1 GiB (or 10 MiB for the
+        # compressors) rather than an awkward fraction of it
+        SMALL, CHUNK, PACK = ("128kiB", 128 * KIB), ("2MiB", 2 * MIB), ("64MiB", 64 * MIB)
         if is_test:
             hash_sizes = comp_sizes = one_size = [SMALL]
             hash_total = comp_total = SMALL[1]
@@ -179,10 +184,10 @@ def do_benchmark_cpu(self, args):
             hash_sizes = [SMALL, CHUNK, PACK]
             comp_sizes = [SMALL, CHUNK]
             one_size = [CHUNK]  # for algorithms where the buffer size does not change behaviour
-            hash_total = 1000 * 1000 * 1000
+            hash_total = GIB
             # compressible data means the codecs actually work, so the slow ones
             # (lzma, high zstd levels) need fewer bytes to stay bearable
-            comp_total = 10 * 1000 * 1000
+            comp_total = 10 * MIB
 
         buffers = {}
 
@@ -225,6 +230,16 @@ def compressible_buffer(nbytes):
                 compressible[nbytes] = bytes(out[:nbytes])
             return compressible[nbytes]
 
+        def data_size_str(size):
+            """The benchmark's own data volumes, always in IEC units.
+
+            These are fixed constants of the benchmark rather than user data, so
+            they do not follow BORG_UNITS - that would make two people's output
+            incomparable and would render binary sizes as awkward decimals
+            (1 GiB as "1.07 GB").
+            """
+            return sizeof_fmt_iec(size, suffix="B", sep=" ", precision=2)
+
         def throughput(size, dt):
             """Rate as MB/s, always - so numbers stay comparable between rows."""
             return f"{size / dt / 1e6:>8.1f} MB/s"
@@ -233,7 +248,7 @@ def report(section, spec, size, dt, width, **extra):
             if args.json:
                 result[section].append({"size": size, "time": dt, **extra})
             else:
-                print(f"{spec:<{width}} {format_file_size(size):<10} {dt:.3f}s  {throughput(size, dt)}")
+                print(f"{spec:<{width}} {data_size_str(size):<11} {dt:.3f}s  {throughput(size, dt)}")
 
         def section_header(section, title):
             if args.json:

From bdf3ab888d7f7b4acb08f5c64f8298a4ff0a15dc Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 23:01:54 +0200
Subject: [PATCH 051/144] benchmark cpu: list the AEAD encryption modes first

The order was historical rather than meaningful. Now the two AEAD modes
come first - aes-256-ocb and chacha20-poly1305, which is what borg
actually uses - followed by the older encrypt-then-MAC ones.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/archiver/benchmark_cmd.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index 31da9a7fe8..315cf74a4a 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -353,26 +353,28 @@ def blake3_hash(d):
 
             section_header("encryption", "Encryption")
             size = chunk_data_size * number_default
+            # AEAD modes first (what borg uses by default), then the legacy
+            # encrypt-then-MAC ones
             tests = [
                 (
-                    "aes-256-ctr-hmac-sha256",
-                    lambda: AES256_CTR_HMAC_SHA256(key_256, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
-                        random_10M, header=b"X"
-                    ),
+                    "aes-256-ocb",
+                    lambda: AES256_OCB(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(random_10M, header=b"X"),
                 ),
                 (
-                    "aes-256-ctr-blake2b",
-                    lambda: AES256_CTR_BLAKE2b(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
+                    "chacha20-poly1305",
+                    lambda: CHACHA20_POLY1305(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(
                         random_10M, header=b"X"
                     ),
                 ),
                 (
-                    "aes-256-ocb",
-                    lambda: AES256_OCB(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(random_10M, header=b"X"),
+                    "aes-256-ctr-hmac-sha256",
+                    lambda: AES256_CTR_HMAC_SHA256(key_256, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
+                        random_10M, header=b"X"
+                    ),
                 ),
                 (
-                    "chacha20-poly1305",
-                    lambda: CHACHA20_POLY1305(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(
+                    "aes-256-ctr-blake2b",
+                    lambda: AES256_CTR_BLAKE2b(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(
                         random_10M, header=b"X"
                     ),
                 ),

From 2a9f691c9319732581be72b3d281c259eaff0d05 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 23:03:59 +0200
Subject: [PATCH 052/144] benchmark cpu: group the compression rows by buffer
 size

They were ordered codec-first, so each codec's two sizes sat together and
comparing codecs meant reading every other line. Now the buffer size is
the outer loop: all codecs at 2MiB, then all at 128kiB.

2MiB comes first because that is the size borg actually compresses - a
typical chunk - and it is the one above zstd's multi-threading threshold.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/archiver/benchmark_cmd.py | 41 ++++++++++++++++--------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index 315cf74a4a..b2cdfdd142 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -389,25 +389,28 @@ def blake3_hash(d):
                 sample = compressible_buffer(comp_sizes[-1][1])
                 ratio = len(sample) / len(CompressionSpec("zstd,3").compressor.compress({}, sample)[1])
                 print(f"(test data is compressible, {ratio:.1f}x at zstd,3)")
-            for spec in [
-                "lz4",
-                "zstd,1",
-                "zstd,3",
-                "zstd,5",
-                "zstd,10",
-                "zstd,16",
-                "zstd,22",
-                "zlib,0",
-                "zlib,6",
-                "zlib,9",
-                "lzma,0",
-                "lzma,6",
-                "lzma,9",
-            ]:
-                compressor = CompressionSpec(spec).compressor
-                for label, nbytes in comp_sizes:
-                    data = compressible_buffer(nbytes)
-                    number = max(3, comp_total // nbytes)  # a few reps even for the fast codecs
+            # grouped by buffer size rather than by codec, so the codecs can be
+            # compared against each other at a glance; chunk-sized first, as
+            # that is what borg actually compresses
+            for label, nbytes in reversed(comp_sizes):
+                data = compressible_buffer(nbytes)
+                number = max(3, comp_total // nbytes)  # a few reps even for the fast codecs
+                for spec in [
+                    "lz4",
+                    "zstd,1",
+                    "zstd,3",
+                    "zstd,5",
+                    "zstd,10",
+                    "zstd,16",
+                    "zstd,22",
+                    "zlib,0",
+                    "zlib,6",
+                    "zlib,9",
+                    "lzma,0",
+                    "lzma,6",
+                    "lzma,9",
+                ]:
+                    compressor = CompressionSpec(spec).compressor
                     dt = timeit(lambda: compressor.compress({}, data), number=number)
                     algo, _, algo_params = spec.partition(",")
                     report(

From c75b1665e41597bec9c7af063f687f2fdef50226 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 23:16:52 +0200
Subject: [PATCH 053/144] benchmark cpu: benchmark zlib,1 rather than zlib,0

zlib level 0 stores rather than compresses - it returns the input
unchanged, ratio 1.00x - so its row measured memcpy speed dressed up as a
compressor. zlib,1 is the fastest level that actually compresses, which
is what the other codecs' lowest rows measure.

Also drops the "(test data is compressible, Nx at zstd,3)" line: it cost
a compression run just to print, and the ratio belongs in the commit that
introduced the test data rather than in every run's output.

lzma,0 stays. Unlike zlib it is not a store mode - it compresses, and on
repetitive input to the same size as lzma,6.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/archiver/benchmark_cmd.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index b2cdfdd142..434aadccab 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -385,10 +385,6 @@ def blake3_hash(d):
 
         if "compression" in selected:
             section_header("compression", "Compression")
-            if not args.json:
-                sample = compressible_buffer(comp_sizes[-1][1])
-                ratio = len(sample) / len(CompressionSpec("zstd,3").compressor.compress({}, sample)[1])
-                print(f"(test data is compressible, {ratio:.1f}x at zstd,3)")
             # grouped by buffer size rather than by codec, so the codecs can be
             # compared against each other at a glance; chunk-sized first, as
             # that is what borg actually compresses
@@ -403,7 +399,7 @@ def blake3_hash(d):
                     "zstd,10",
                     "zstd,16",
                     "zstd,22",
-                    "zlib,0",
+                    "zlib,1",
                     "zlib,6",
                     "zlib,9",
                     "lzma,0",

From d927fc75c9d1c07b6af261c68046be266c1e9a6f Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 6 Aug 2026 23:21:45 +0200
Subject: [PATCH 054/144] benchmark cpu: order the hashes fastest first

blake3 leads, at pack size then chunk size, followed by hmac-sha256 and
blake2b-256 at chunk size:

    blake3 (64MiB)             1.00 GiB    0.063s   17147.6 MB/s
    blake3 (2MiB)              1.00 GiB    0.088s   12252.2 MB/s
    hmac-sha256 (2MiB)         1.00 GiB    0.352s    3050.6 MB/s
    blake2b-256 (2MiB)         1.00 GiB    1.003s    1070.3 MB/s

Drops the 128kiB blake3 row. It sat below blake3's multi-threading
threshold, so it measured the single-threaded path - true to what borg
does with a chunk that small, but not what the section is for now that
the remaining rows are all multi-threaded and directly comparable.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/archiver/benchmark_cmd.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index 434aadccab..f82e3c65ca 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -181,7 +181,8 @@ def do_benchmark_cpu(self, args):
             hash_sizes = comp_sizes = one_size = [SMALL]
             hash_total = comp_total = SMALL[1]
         else:
-            hash_sizes = [SMALL, CHUNK, PACK]
+            # blake3: a borg pack, then a borg chunk - both above its MT threshold
+            hash_sizes = [PACK, CHUNK]
             comp_sizes = [SMALL, CHUNK]
             one_size = [CHUNK]  # for algorithms where the buffer size does not change behaviour
             hash_total = GIB
@@ -334,11 +335,11 @@ def blake3_hash(d):
 
             section_header("hashes", "Cryptographic hashes / MACs")
             # only blake3 uses multiple threads, and only above a size threshold,
-            # so it is the one worth measuring at several buffer sizes.
+            # so it is the one worth measuring at more than one buffer size.
             hashes_tests = [
+                ("blake3", hash_sizes, blake3_hash),
                 ("hmac-sha256", one_size, lambda d: hmac_sha256(key_256, d)),
                 ("blake2b-256", one_size, lambda d: blake2b_256(key_256, d)),
-                ("blake3", hash_sizes, blake3_hash),
             ]
             for spec, sizes, func in hashes_tests:
                 for label, nbytes in sizes:

From 46dc79f6b1da46038a26645f24bf4ab86c9a3a7f Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 7 Aug 2026 10:42:06 +0200
Subject: [PATCH 055/144] benchmark cpu: fix the help text, align the columns

The epilog still described earlier revisions of this branch: 100kiB and
50MiB rather than the 128kiB and 64MiB the code uses, hashes measured
below the multi-threading threshold when that row is gone (they are now
measured at pack and chunk size, both above it), and "every row processes
roughly the same total number of bytes" when the compressors do 10 MiB
and everything else 1 GiB. It also no longer said anywhere that the
compression section runs on compressible data, without which its numbers
cannot be read. This text ends up in the man page, so it should be right.

The spec column was 26 characters wide in three sections and 20 in the
other two, and the msgpack row padded its size field to 10 where the rest
use 11, so nothing lined up between sections:

    blake3 (64MiB)             1.00 GiB    0.062s   17253.6 MB/s
    msgpack              128k Items 0.051s    2515.6 kItems/s

One WIDTH constant now covers all of them - 26 fits the longest spec of
any section, "buzhash64,19,23,21,4095,2".

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/borg/archiver/benchmark_cmd.py | 45 +++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 17 deletions(-)

diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index f82e3c65ca..17487ea490 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -164,18 +164,23 @@ def do_benchmark_cpu(self, args):
 
         KIB, MIB, GIB = 1024, 1024 * 1024, 1024 * 1024 * 1024
 
+        # width of the spec column, wide enough for the longest row of any section, so
+        # that all sections line up with each other
+        WIDTH = 26
+
         is_test = "_BORG_BENCHMARK_CPU_TEST" in os.environ
         # Use minimal data and one iteration in test mode to keep CI fast.
         chunk_data_size = 128 * KIB if is_test else 8 * MIB
         number_default = 1 if is_test else GIB // chunk_data_size
 
         # Buffer sizes matter where an algorithm switches to multiple threads:
-        # blake3 and zstd do so above a threshold, so a size below it, one the
-        # size of a typical borg chunk and one the size of a borg pack behave
-        # quite differently. Every row processes the same total either way, so
-        # the throughput column stays comparable across sizes.
-        # powers of two, so every row processes exactly 1 GiB (or 10 MiB for the
-        # compressors) rather than an awkward fraction of it
+        # blake3 and zstd do so above a threshold, so a buffer below it and one
+        # above it behave quite differently. The hashes are measured at pack and
+        # chunk size (both above blake3's threshold), the compressors at chunk
+        # size and below zstd's threshold. Within a section every row processes
+        # the same total, so the throughput column stays comparable across sizes.
+        # All powers of two, so that total is exactly 1 GiB (or 10 MiB for the
+        # compressors) rather than an awkward fraction of it.
         SMALL, CHUNK, PACK = ("128kiB", 128 * KIB), ("2MiB", 2 * MIB), ("64MiB", 64 * MIB)
         if is_test:
             hash_sizes = comp_sizes = one_size = [SMALL]
@@ -245,11 +250,11 @@ def throughput(size, dt):
             """Rate as MB/s, always - so numbers stay comparable between rows."""
             return f"{size / dt / 1e6:>8.1f} MB/s"
 
-        def report(section, spec, size, dt, width, **extra):
+        def report(section, spec, size, dt, **extra):
             if args.json:
                 result[section].append({"size": size, "time": dt, **extra})
             else:
-                print(f"{spec:<{width}} {data_size_str(size):<11} {dt:.3f}s  {throughput(size, dt)}")
+                print(f"{spec:<{WIDTH}} {data_size_str(size):<11} {dt:.3f}s  {throughput(size, dt)}")
 
         def section_header(section, title):
             if args.json:
@@ -320,7 +325,7 @@ def chunkit(ch):
             ]:
                 dt = timeit(func, setup, number=number_default, globals=vars)
                 algo, _, algo_params = spec.partition(",")
-                report("chunkers", spec, size, dt, 26, algo=algo, algo_params=algo_params)
+                report("chunkers", spec, size, dt, algo=algo, algo_params=algo_params)
 
         if "hashing" in selected:
             from ..crypto.low_level import hmac_sha256, blake2b_256
@@ -346,7 +351,7 @@ def blake3_hash(d):
                     data = buffer(nbytes)
                     number = max(1, hash_total // nbytes)
                     dt = timeit(lambda: func(data), number=number)
-                    report("hashes", f"{spec} ({label})", nbytes * number, dt, 26, algo=spec, buffer_size=nbytes)
+                    report("hashes", f"{spec} ({label})", nbytes * number, dt, algo=spec, buffer_size=nbytes)
 
         if "encryption" in selected:
             from ..crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
@@ -382,7 +387,7 @@ def blake3_hash(d):
             ]
             for spec, func in tests:
                 dt = timeit(func, number=number_default)
-                report("encryption", spec, size, dt, 26, algo=spec)
+                report("encryption", spec, size, dt, algo=spec)
 
         if "compression" in selected:
             section_header("compression", "Compression")
@@ -415,7 +420,6 @@ def blake3_hash(d):
                         f"{spec} ({label})",
                         nbytes * number,
                         dt,
-                        20,
                         algo=algo,
                         algo_params=algo_params,
                         buffer_size=nbytes,
@@ -433,7 +437,7 @@ def blake3_hash(d):
             else:
                 # this one packs items, not bytes, so it gets a rate in its own unit
                 size = "%dk Items" % (count // 1000)
-                print(f"{spec:<20} {size:<10} {dt:.3f}s  {count / dt / 1000:>8.1f} kItems/s")
+                print(f"{spec:<{WIDTH}} {size:<11} {dt:.3f}s  {count / dt / 1000:>8.1f} kItems/s")
 
         if args.json:
             json_print(result)
@@ -517,10 +521,17 @@ def build_parser_benchmarks(self, subparsers, common_parser, mid_common_parser):
         --encryption, --compression, --msgpack to run only those.
 
         Some algorithms use multiple threads only above a size threshold, so the
-        hashes and compressors are measured at several buffer sizes: 100kiB (below
-        the threshold), 2MiB (a typical borg chunk) and, for hashes, 50MiB (a borg
-        pack). Every row processes roughly the same total number of bytes, so the
-        throughput column is comparable between them.
+        hashes and the compressors are measured at more than one buffer size: the
+        hashes at 64MiB (a borg pack) and 2MiB (a typical borg chunk), both above
+        blake3's threshold, the compressors at 2MiB and 128kiB, which is below
+        zstd's. Within a section every row processes the same total number of
+        bytes - 1 GiB, or 10 MiB for the compressors - so the throughput column is
+        comparable between rows.
+
+        The compressors work on synthetic text-like data that compresses about 4x
+        at zstd,3. Random data would be the worst possible input: no codec can
+        compress it, so all of them would take their incompressible fast path and
+        the levels would barely differ.
         """
         )
         subparser = ArgumentParser(

From 2b3953868299b598f7d40a55a7a4fc7e03aac521 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 7 Aug 2026 11:11:51 +0200
Subject: [PATCH 056/144] docs: document the chunker scan kernel env vars

BORG_FASTCDC_KERNEL, BORG_BUZHASH64_KERNEL and BORG_AES_CHUNKER_KERNEL were
only mentioned in the changelog, while every other tunable is listed in the
environment variables docs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/usage/general/environment.rst.inc | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc
index bafba45636..49f16d5622 100644
--- a/docs/usage/general/environment.rst.inc
+++ b/docs/usage/general/environment.rst.inc
@@ -182,6 +182,32 @@ General:
         +0.05% archive size for 1MiB chunks, +0.64% for 8MiB ones, more at higher levels), and
         it uses more cpu time in total to reduce the wallclock time. Set it to 1 if you would
         rather have the smaller archive, or if borg has to share the cpu with other work.
+    BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL
+        Select the scan kernel the ``fastcdc`` / ``buzhash64`` chunker uses (default:
+        ``scalar``, the plain sequential loop). Accepted values are ``avx512``, ``avx2``,
+        ``neon``, ``blockwise`` and ``scalar``.
+        All kernels chunk identically - same cut points, same chunk ids - and differ only in
+        speed, so this is safe to change at any time, also for an existing repository.
+        Which kernel is fastest is not predictable from the instruction set: it depends on the
+        cpu and on the compiler that built borg, and the sequential loop wins on some machines.
+        Nothing is selected automatically, so measure on your own hardware with
+        ``borg benchmark cpu --chunking`` before setting these.
+        ``avx512`` and ``avx2`` exist only on x86-64, ``neon`` only on aarch64, and only if the
+        compiler that built borg supported them; ``scalar`` and ``blockwise`` are portable C
+        and always available.
+        Requesting a kernel that this build or this cpu cannot run is an error rather than a
+        silent fallback, so a benchmark can not accidentally measure a different kernel.
+        ``borg create --debug`` logs the chunker and the kernel it was created with.
+    BORG_AES_CHUNKER_KERNEL
+        Select the scan kernel used by the AES based chunkers - one variable for all three of
+        ``toeplitz-aes``, ``rabin-aes`` and ``goldilocks-aes`` (default: ``evp``, the portable
+        OpenSSL path). Accepted values are ``vaes``, ``aes-ni``, ``aes-arm64`` and ``evp``.
+        As with the chunker kernels above, all of them chunk identically and differ only in
+        speed, nothing is selected automatically, and a kernel that can not run here is an
+        error rather than a silent fallback.
+        ``vaes`` and ``aes-ni`` exist only on x86-64, ``aes-arm64`` only on aarch64.
+        ``vaes`` additionally needs a compiler that knows it (gcc >= 11 / clang >= 14), so a
+        cpu supporting VAES is not by itself enough to have that kernel available.
     BORG_SHOW_SYSINFO
         When set to no (default: yes), system information (like OS, Python version, ...) in
         exceptions is not shown.

From 896e8f587b021dbe9d8c84a949081c51991229cd Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 7 Aug 2026 11:28:09 +0200
Subject: [PATCH 057/144] benchmark cpu: name the section flags consistently

--encryption / --compression / --msgpack become --encrypting /
--compressing / --msgpacking, so all five section flags are gerunds like
the already existing --chunking and --hashing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 docs/changes.rst                                 |  4 ++--
 src/borg/archiver/benchmark_cmd.py               | 16 ++++++++--------
 .../testsuite/archiver/benchmark_cmd_test.py     |  6 +++---
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 0dc7d96874..e6357389c4 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -202,8 +202,8 @@ New features:
 - version: add --json output, #10004
 - benchmark cpu: add a throughput column (MB/s), #10049
 - benchmark cpu: measure hashes and compressors at several buffer sizes, and
-  add --chunking / --hashing / --encryption / --compression / --msgpack to run
-  only a subset, #10050
+  add --chunking / --hashing / --encrypting / --compressing / --msgpacking to
+  run only a subset, #10050
 - benchmark cpu: compress deterministic compressible data instead of random noise
 - benchmark cpu: measure blake3 the way borg uses it (multi-threaded only above
   BORG_BLAKE3_MT_THRESHOLD)
diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py
index 17487ea490..614640adf0 100644
--- a/src/borg/archiver/benchmark_cmd.py
+++ b/src/borg/archiver/benchmark_cmd.py
@@ -157,7 +157,7 @@ def do_benchmark_cpu(self, args):
         result = {} if args.json else None
 
         # no section selected means: run them all
-        sections = ("chunking", "hashing", "encryption", "compression", "msgpack")
+        sections = ("chunking", "hashing", "encrypting", "compressing", "msgpacking")
         selected = {name for name in sections if getattr(args, name)}
         if not selected:
             selected = set(sections)
@@ -353,7 +353,7 @@ def blake3_hash(d):
                     dt = timeit(lambda: func(data), number=number)
                     report("hashes", f"{spec} ({label})", nbytes * number, dt, algo=spec, buffer_size=nbytes)
 
-        if "encryption" in selected:
+        if "encrypting" in selected:
             from ..crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256
             from ..crypto.low_level import AES256_OCB, CHACHA20_POLY1305
 
@@ -389,7 +389,7 @@ def blake3_hash(d):
                 dt = timeit(func, number=number_default)
                 report("encryption", spec, size, dt, algo=spec)
 
-        if "compression" in selected:
+        if "compressing" in selected:
             section_header("compression", "Compression")
             # grouped by buffer size rather than by codec, so the codecs can be
             # compared against each other at a glance; chunk-sized first, as
@@ -425,7 +425,7 @@ def blake3_hash(d):
                         buffer_size=nbytes,
                     )
 
-        if "msgpack" in selected:
+        if "msgpacking" in selected:
             section_header("msgpack", "msgpack")
             item = Item(path="foo/bar/baz", mode=660, mtime=1234567)
             items = [item.as_dict()] * 1000
@@ -518,7 +518,7 @@ def build_parser_benchmarks(self, subparsers, common_parser, mid_common_parser):
         - enough free memory so there will be no slow down due to paging activity
 
         By default all benchmarks run. Give one or more of --chunking, --hashing,
-        --encryption, --compression, --msgpack to run only those.
+        --encrypting, --compressing, --msgpacking to run only those.
 
         Some algorithms use multiple threads only above a size threshold, so the
         hashes and the compressors are measured at more than one buffer size: the
@@ -541,6 +541,6 @@ def build_parser_benchmarks(self, subparsers, common_parser, mid_common_parser):
         subparser.add_argument("--json", action="store_true", help="format output as JSON")
         subparser.add_argument("--chunking", action="store_true", help="benchmark the chunkers")
         subparser.add_argument("--hashing", action="store_true", help="benchmark the hashes / MACs")
-        subparser.add_argument("--encryption", action="store_true", help="benchmark the encryption modes")
-        subparser.add_argument("--compression", action="store_true", help="benchmark the compressors")
-        subparser.add_argument("--msgpack", action="store_true", help="benchmark msgpack item packing")
+        subparser.add_argument("--encrypting", action="store_true", help="benchmark the encryption modes")
+        subparser.add_argument("--compressing", action="store_true", help="benchmark the compressors")
+        subparser.add_argument("--msgpacking", action="store_true", help="benchmark msgpack item packing")
diff --git a/src/borg/testsuite/archiver/benchmark_cmd_test.py b/src/borg/testsuite/archiver/benchmark_cmd_test.py
index e04021ab02..f4c16185c9 100644
--- a/src/borg/testsuite/archiver/benchmark_cmd_test.py
+++ b/src/borg/testsuite/archiver/benchmark_cmd_test.py
@@ -63,9 +63,9 @@ def test_benchmark_cpu(archiver, monkeypatch):
     [
         ("--chunking", "Chunkers"),
         ("--hashing", "Cryptographic hashes / MACs"),
-        ("--encryption", "Encryption"),
-        ("--compression", "Compression"),
-        ("--msgpack", "msgpack"),
+        ("--encrypting", "Encryption"),
+        ("--compressing", "Compression"),
+        ("--msgpacking", "msgpack"),
     ],
 )
 def test_benchmark_cpu_section_selection(archiver, monkeypatch, flag, header):

From 8f2de9e809565d7099efc7b07d01f7c91c86978a Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 7 Aug 2026 12:57:28 +0200
Subject: [PATCH 058/144] tests: cover every scan kernel, not just the default
 one

The kernel identity tests looped over the kernels inside one test function,
so the first disagreement hid the rest, the failing kernel was not in the
test id, and kernels that a build or CPU cannot run were silently skipped -
nothing said which kernels a machine had actually covered.

They are parametrized now, and the shared chunker_with_kernel() helper turns
an unusable kernel into a pytest skip carrying the chunker's own error
message, so "pytest -rs" lists exactly what was and was not exercised. The
"a portable kernel must exist here" assertions become their own tests, since
scalar/blockwise/evp not being selectable is a bug rather than a limited
platform.

The fuzz tests now run per kernel as well. They are what covers odd sizes,
all-zero and all-same data, i.e. the block tails and sub-block inputs where a
vector kernel would break - and they only ever ran against the default kernel,
which since 4944f7474 is the sequential one. CI does not set BORG_TESTS_SLOW,
so this costs CI nothing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TwV6jAxVttxcnJ5AVhfPxm
---
 src/borg/testsuite/chunkers/__init__.py       | 20 +++++++
 src/borg/testsuite/chunkers/buzhash64_test.py | 47 ++++++++-------
 src/borg/testsuite/chunkers/fastcdc_test.py   | 57 +++++++++++--------
 .../testsuite/chunkers/phte_chunkers_test.py  | 40 +++++++------
 4 files changed, 103 insertions(+), 61 deletions(-)

diff --git a/src/borg/testsuite/chunkers/__init__.py b/src/borg/testsuite/chunkers/__init__.py
index ce9ff11e21..7dd2d7ac9d 100644
--- a/src/borg/testsuite/chunkers/__init__.py
+++ b/src/borg/testsuite/chunkers/__init__.py
@@ -1,11 +1,31 @@
 import os
 import tempfile
 
+import pytest
+
 from borg.constants import *  # noqa
 
 from ...chunkers import has_seek_hole
 
 
+def chunker_with_kernel(monkeypatch, envvar, kernel, make):
+    """Build a chunker with <envvar> set to <kernel>, or skip if it cannot run here.
+
+    Which kernels exist depends on the build (compiler support) and on the CPU, so a
+    kernel that cannot be selected is not a failure - the test is skipped, carrying
+    the reason the chunker gave, so `pytest -rs` shows which kernels a machine covered.
+    Requesting an unusable kernel raising (rather than falling back) is itself tested,
+    see test_kernel_env_rejects_unusable.
+    """
+    monkeypatch.setenv(envvar, kernel)
+    try:
+        chunker = make()
+    except ValueError as err:
+        pytest.skip(str(err))
+    assert chunker.kernel == kernel  # we must have got what we asked for, not a fallback
+    return chunker
+
+
 def cf(chunks):
     """Chunk filter."""
 
diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py
index 202d32ed81..66adcfc701 100644
--- a/src/borg/testsuite/chunkers/buzhash64_test.py
+++ b/src/borg/testsuite/chunkers/buzhash64_test.py
@@ -5,7 +5,7 @@
 
 import pytest
 
-from . import cf, cf_expand
+from . import cf, cf_expand, chunker_with_kernel
 from ...chunkers import ChunkerBuzHash64
 from ...chunkers.buzhash64 import buzhash64_get_table
 from ...constants import *  # NOQA
@@ -103,10 +103,16 @@ def test_buzhash64_table():
         assert bit_count == 128  # 50% of 256 = 128
 
 
+ALL_BUZHASH64_KERNELS = ("neon", "avx512", "avx2", "blockwise", "scalar")
+
+
 @pytest.mark.skipif("BORG_TESTS_SLOW" not in os.environ, reason="slow tests not enabled, use BORG_TESTS_SLOW=1")
+@pytest.mark.parametrize("kernel", ALL_BUZHASH64_KERNELS)
 @pytest.mark.parametrize("worker", range(os.cpu_count() or 1))
-def test_fuzz_bh64(worker):
+def test_fuzz_bh64(worker, kernel, monkeypatch):
     # Fuzz buzhash64 with random and uniform data of misc. sizes and misc keys.
+    # Every kernel gets fuzzed - see the fastcdc counterpart for why the odd
+    # sizes here matter more to the vector kernels than the identity test does.
     def rnd_key():
         return os.urandom(32)
 
@@ -118,7 +124,12 @@ def rnd_key():
     sizes = [random.randint(1, 4 * 1024 * 1024) for _ in range(50)]
 
     for key in keys:
-        chunker = ChunkerBuzHash64(key, min_exp, max_exp, mask_bits, win_size, nc_level)
+        chunker = chunker_with_kernel(
+            monkeypatch,
+            "BORG_BUZHASH64_KERNEL",
+            kernel,
+            lambda: ChunkerBuzHash64(key, min_exp, max_exp, mask_bits, win_size, nc_level),
+        )
         for size in sizes:
             # Random data
             data = os.urandom(size)
@@ -142,10 +153,8 @@ def rnd_key():
             assert reconstructed == data
 
 
-ALL_BUZHASH64_KERNELS = ("neon", "avx512", "avx2", "blockwise", "scalar")
-
-
-def test_buzhash64_kernels_identical(monkeypatch):
+@pytest.mark.parametrize("kernel", ALL_BUZHASH64_KERNELS)
+def test_buzhash64_kernels_identical(kernel, monkeypatch):
     # Every scan kernel this platform accepts must produce identical cut points;
     # see the fastcdc counterpart. On aarch64 this is what keeps the NEON kernel
     # verified - it is selectable but never auto-selected.
@@ -158,15 +167,15 @@ def sizes(chunker):
     monkeypatch.delenv("BORG_BUZHASH64_KERNEL", raising=False)
     reference = sizes(ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2))
 
-    tested = []
-    for name in ALL_BUZHASH64_KERNELS:
-        monkeypatch.setenv("BORG_BUZHASH64_KERNEL", name)
-        try:
-            chunker = ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
-        except ValueError:
-            continue  # not available on this build/CPU
-        assert chunker.kernel == name
-        assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one"
-        tested.append(name)
-
-    assert "blockwise" in tested and "scalar" in tested
+    chunker = chunker_with_kernel(
+        monkeypatch, "BORG_BUZHASH64_KERNEL", kernel, lambda: ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2)
+    )
+    assert sizes(chunker) == reference, f"kernel {kernel} disagrees with the default one"
+
+
+@pytest.mark.parametrize("kernel", ["blockwise", "scalar"])
+def test_buzhash64_portable_kernel_available(kernel, monkeypatch):
+    # portable C, so these two must be selectable everywhere - see the fastcdc counterpart.
+    key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
+    monkeypatch.setenv("BORG_BUZHASH64_KERNEL", kernel)
+    assert ChunkerBuzHash64(key0, 10, 16, 14, 4095, 2).kernel == kernel
diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py
index 1a7eaaf2e3..bb8bdb6a4b 100644
--- a/src/borg/testsuite/chunkers/fastcdc_test.py
+++ b/src/borg/testsuite/chunkers/fastcdc_test.py
@@ -5,7 +5,7 @@
 
 import pytest
 
-from . import cf, cf_expand
+from . import cf, cf_expand, chunker_with_kernel
 from ...chunkers import ChunkerFastCDC, get_chunker
 from ...chunkers.fastcdc import fastcdc_get_gear_table
 from ...constants import *  # NOQA
@@ -142,10 +142,17 @@ def test_fastcdc_params_parsing():
         ChunkerParams("fastcdc,20,20,20,2")
 
 
+ALL_FASTCDC_KERNELS = ("neon", "avx512", "avx2", "blockwise", "scalar")
+
+
 @pytest.mark.skipif("BORG_TESTS_SLOW" not in os.environ, reason="slow tests not enabled, use BORG_TESTS_SLOW=1")
+@pytest.mark.parametrize("kernel", ALL_FASTCDC_KERNELS)
 @pytest.mark.parametrize("worker", range(os.cpu_count() or 1))
-def test_fuzz_fastcdc(worker):
+def test_fuzz_fastcdc(worker, kernel, monkeypatch):
     # Fuzz fastcdc with random and uniform data of misc. sizes and misc keys.
+    # Every kernel gets fuzzed: the sizes here are not multiples of a block, so
+    # this is what covers the vector kernels' tails and their sub-block inputs,
+    # which the fixed-size buffer in test_fastcdc_kernels_identical never reaches.
     def rnd_key():
         return os.urandom(32)
 
@@ -157,7 +164,12 @@ def rnd_key():
     sizes = [random.randint(1, 4 * 1024 * 1024) for _ in range(50)]
 
     for key in keys:
-        chunker = ChunkerFastCDC(key, min_exp, max_exp, mask_bits, nc_level)
+        chunker = chunker_with_kernel(
+            monkeypatch,
+            "BORG_FASTCDC_KERNEL",
+            kernel,
+            lambda: ChunkerFastCDC(key, min_exp, max_exp, mask_bits, nc_level),
+        )
         for size in sizes:
             # Random data
             data = os.urandom(size)
@@ -178,16 +190,13 @@ def rnd_key():
             assert b"".join(parts) == data
 
 
-ALL_FASTCDC_KERNELS = ("neon", "avx512", "avx2", "blockwise", "scalar")
-
-
-def test_fastcdc_kernels_identical(monkeypatch):
+@pytest.mark.parametrize("kernel", ALL_FASTCDC_KERNELS)
+def test_fastcdc_kernels_identical(kernel, monkeypatch):
     # Every scan kernel this platform accepts must produce identical cut points.
-    # Kernels this build/CPU cannot run raise and are skipped, so the same test
-    # covers whatever tier the machine happens to have - including kernels that
-    # exist but are not auto-selected, which nothing else would exercise.
+    # Kernels this build/CPU cannot run are skipped, so the same test covers
+    # whatever tier the machine happens to have - including kernels that exist
+    # but are not auto-selected, which nothing else would exercise.
     data = os.urandom(4 * 1024 * 1024)
-    key0 = hex_to_bin("ad9f89095817f0566337dc9ee292fcd59b70f054a8200151f1df5f21704824da")
 
     def sizes(chunker):
         return [c.meta["size"] for c in chunker.chunkify(BytesIO(data))]
@@ -195,19 +204,19 @@ def sizes(chunker):
     monkeypatch.delenv("BORG_FASTCDC_KERNEL", raising=False)
     reference = sizes(ChunkerFastCDC(key0, 10, 16, 14, 2))
 
-    tested = []
-    for name in ALL_FASTCDC_KERNELS:
-        monkeypatch.setenv("BORG_FASTCDC_KERNEL", name)
-        try:
-            chunker = ChunkerFastCDC(key0, 10, 16, 14, 2)
-        except ValueError:
-            continue  # not available on this build/CPU
-        assert chunker.kernel == name
-        assert sizes(chunker) == reference, f"kernel {name} disagrees with the default one"
-        tested.append(name)
-
-    # the portable kernels exist everywhere; if they were skipped, the loop is broken
-    assert "blockwise" in tested and "scalar" in tested
+    chunker = chunker_with_kernel(
+        monkeypatch, "BORG_FASTCDC_KERNEL", kernel, lambda: ChunkerFastCDC(key0, 10, 16, 14, 2)
+    )
+    assert sizes(chunker) == reference, f"kernel {kernel} disagrees with the default one"
+
+
+@pytest.mark.parametrize("kernel", ["blockwise", "scalar"])
+def test_fastcdc_portable_kernel_available(kernel, monkeypatch):
+    # scalar and blockwise are portable C and exist in every build on every CPU.
+    # If one of them cannot be selected, kernel selection is broken rather than
+    # the platform being limited - so this must not skip the way the test above does.
+    monkeypatch.setenv("BORG_FASTCDC_KERNEL", kernel)
+    assert ChunkerFastCDC(key0, 10, 16, 14, 2).kernel == kernel
 
 
 @pytest.mark.parametrize("envvar", ["BORG_FASTCDC_KERNEL", "BORG_BUZHASH64_KERNEL", "BORG_AES_CHUNKER_KERNEL"])
diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py
index a231df6002..dda8354e2b 100644
--- a/src/borg/testsuite/chunkers/phte_chunkers_test.py
+++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py
@@ -14,7 +14,7 @@
 
 import pytest
 
-from . import cf, cf_expand
+from . import cf, cf_expand, chunker_with_kernel
 from ...chunkers import ChunkerRabinAES, ChunkerGoldilocksAES, ChunkerToeplitzAES, get_chunker
 from ...constants import *  # NOQA
 from ...helpers import hex_to_bin
@@ -47,10 +47,11 @@ def chunker_spec(request):
 ALL_AES_KERNELS = ("vaes", "aes-ni", "aes-arm64", "evp")
 
 
-def test_kernels_identical(chunker_spec, monkeypatch):
+@pytest.mark.parametrize("kernel", ALL_AES_KERNELS)
+def test_kernels_identical(chunker_spec, kernel, monkeypatch):
     # Every scan path this platform accepts - OpenSSL EVP, the 128-bit AES
     # instructions, VAES - must produce identical cut points. Paths this
-    # build/CPU cannot run raise and are skipped.
+    # build/CPU cannot run are skipped.
     cls, algo, params = chunker_spec
     data = os.urandom(4 * 1024 * 1024)
 
@@ -60,19 +61,17 @@ def sizes(chunker):
     monkeypatch.delenv(KERNEL_ENV, raising=False)
     reference = sizes(cls(key0, 10, 16, 14, 2))
 
-    tested = []
-    for name in ALL_AES_KERNELS:
-        monkeypatch.setenv(KERNEL_ENV, name)
-        try:
-            chunker = cls(key0, 10, 16, 14, 2)
-        except ValueError:
-            continue  # not available on this build/CPU
-        assert chunker.kernel == name
-        assert sizes(chunker) == reference, f"path {name} disagrees with the default one"
-        tested.append(name)
+    chunker = chunker_with_kernel(monkeypatch, KERNEL_ENV, kernel, lambda: cls(key0, 10, 16, 14, 2))
+    assert sizes(chunker) == reference, f"path {kernel} disagrees with the default one"
 
-    # the portable OpenSSL path exists everywhere
-    assert "evp" in tested
+
+def test_evp_kernel_available(chunker_spec, monkeypatch):
+    # EVP is the portable OpenSSL path: it exists in every build on every CPU, so
+    # unlike the test above this must not skip - if it cannot be selected, path
+    # selection is broken rather than the platform being limited.
+    cls, algo, params = chunker_spec
+    monkeypatch.setenv(KERNEL_ENV, "evp")
+    assert cls(key0, 10, 16, 14, 2).kernel == "evp"
 
 
 def test_chunksize_distribution(chunker_spec):
@@ -157,9 +156,12 @@ def test_params_parsing(chunker_spec):
 
 
 @pytest.mark.skipif("BORG_TESTS_SLOW" not in os.environ, reason="slow tests not enabled, use BORG_TESTS_SLOW=1")
+@pytest.mark.parametrize("kernel", ALL_AES_KERNELS)
 @pytest.mark.parametrize("worker", range(os.cpu_count() or 1))
-def test_fuzz(chunker_spec, worker):
-    # Fuzz with random and uniform data of misc. sizes and misc keys.
+def test_fuzz(chunker_spec, worker, kernel, monkeypatch):
+    # Fuzz with random and uniform data of misc. sizes and misc keys, on every
+    # scan path: these odd sizes are what exercise the hardware paths' tails and
+    # their groups-of-8 handling, which test_kernels_identical does not reach.
     cls, algo, params = chunker_spec
 
     # decompose <ALGO>_PARAMS = (algo, min_exp, max_exp, mask_bits, nc_level)
@@ -170,7 +172,9 @@ def test_fuzz(chunker_spec, worker):
     sizes = [random.randint(1, 4 * 1024 * 1024) for _ in range(50)]
 
     for key in keys:
-        chunker = cls(key, min_exp, max_exp, mask_bits, nc_level)
+        chunker = chunker_with_kernel(
+            monkeypatch, KERNEL_ENV, kernel, lambda: cls(key, min_exp, max_exp, mask_bits, nc_level)
+        )
         for size in sizes:
             # Random data
             data = os.urandom(size)

From 9cf947fe821ce1141327f4cc7883b81ba2c76a7c Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 7 Aug 2026 15:30:35 +0200
Subject: [PATCH 059/144] update CHANGES

---
 docs/changes.rst | 64 +++++++++++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 28 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 35a2d798f6..16ce70d418 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -167,23 +167,22 @@ Version 2.0.0b23 (not released yet)
 
 New features:
 
-- faster:
-
-  - create:
-
-    - use multi-threaded zstd compression for big chunks, #9961
-    - use multi-threaded blake3 hashing for big chunks, #9958
-    - overlap pack hash/store and build of the next pack, #9988.
-      BORG_PACK_ASYNC=no disables the store-thread (debugging aid).
-    - chunkers, crypto: release the GIL in pure-C hot paths
-  - extract / mount:
-
-    - avoid refetching/reparsing repeated chunks, #1678
-    - do not verify the chunk id on every read from an encrypted repo (the
-      AEAD authentication covers reads), see BORG_ASSERT_ID, #9994, #7362
-    - serve all-zero chunks without repository access; also cache
-      recently parsed chunks, #1678
-- Chunkers:
+- faster create:
+
+  - use multi-threaded zstd compression for big chunks, #9961
+  - use multi-threaded blake3 hashing for big chunks, #9958
+  - overlap pack hash/store and build of the next pack, #9988.
+    BORG_PACK_ASYNC=no disables the store-thread (debugging aid).
+  - chunkers, crypto: release the GIL in pure-C hot paths
+  - give each thread its own LZ4 scratch buffer, #10032
+- faster extract / mount:
+
+  - avoid refetching/reparsing repeated chunks, #1678
+  - do not verify the chunk id on every read from an encrypted repo (the
+    AEAD authentication covers reads), see BORG_ASSERT_ID, #9994, #7362
+  - serve all-zero chunks without repository access; also cache
+    recently parsed chunks, #1678
+- more, faster, and more secure chunkers:
 
   - fastcdc is the new and faster default chunker, #9957
   - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043:
@@ -195,26 +194,35 @@ New features:
       which one is fastest depends on the CPU *and* the compiler, so nothing
       is chosen automatically
   - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers
-    (UHF-then-PRF), with direct AES hw acceleration or via OpenSSL, #9987
-  - toeplitz-aes, rabin-aes, goldilocks-aes: VAES/AVX-512 scan path on x86-64
-    (4 AES blocks per instruction), #10043
+    (UHF-then-PRF), with direct AES hw acceleration (AES-NI or VAES/AVX-512) or
+    via OpenSSL, #9987, #10043
   - zero-copy fill and lazy buffer compaction optimizations
+  - log the chunker and its scan kernel at debug level
 - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice
   replacement for `borg mount` in some use cases, #9942
 - mount: expose POSIX ACLs on Linux mounts (not enforced), #1042
 - analyze: report deduplicated size of a set of archives, #5741
 - repo-compress: was temporarily gone, re-added now with pack support, #9663
 - version: add --json output, #10004
-- benchmark cpu: add a throughput column (MB/s), #10049
-- benchmark cpu: measure hashes and compressors at several buffer sizes, and
-  add --chunking / --hashing / --encrypting / --compressing / --msgpacking to
-  run only a subset, #10050
-- benchmark cpu: compress deterministic compressible data instead of random noise
-- benchmark cpu: measure blake3 the way borg uses it (multi-threaded only above
-  BORG_BLAKE3_MT_THRESHOLD)
-- completion: generate fish completions, remove hand-written ones, #9989
+- benchmark cpu:
+
+  - add a throughput column (MB/s), #10049
+  - measure hashes and compressors at several buffer sizes
+  - compress deterministic compressible data instead of random noise
+  - measure algorithms the way borg uses them (e.g. multithreading on/off
+    depending on data size)
+  - use --chunking / --hashing / --encrypting / --compressing / --msgpacking
+    to run only a subset of the benchmarks, #10050
+- completion: generate fish and tcsh completions, #9989, #9503
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
+- check:
+
+  - keep pack check results, add --max-age to reuse them, #9696, #9925
+  - report missing chunks grouped as chunk -> files -> archives, #9218, #9965
+  - calendar-aware --max-age, symmetric clock-skew window
+  - check: stream one line per missing chunk id, run report on abort, lower report caps, #9218
+
 
 Fixes:
 

From 7839e9f9fede5336a3c6eaf6fddaf8912d8a1cd3 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Sat, 8 Aug 2026 17:48:22 +0530
Subject: [PATCH 060/144] check: honest per-run interrupt count, drop redundant
 save, fix stale SIGINT docs

---
 src/borg/archiver/check_cmd.py | 6 +++---
 src/borg/repository.py         | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index db171f6729..1c1fa2e80a 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -192,9 +192,9 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         next safe boundary, leaving the repository and its chunk index in a consistent state.
         The repository check stops after the current pack; ``--verify-data`` and
         ``--find-lost-archives`` stop after the current chunk; a ``--repair`` archive check
-        stops between whole archives. A partial repository check (``--max-duration``) saves its
-        progress so a later partial check resumes where it stopped; a full check restarts from
-        the beginning. With ``--repair``, an interrupted archive check may leave some archives
+        stops between whole archives. Results recorded before the interrupt are kept, so a later
+        check does not re-verify those packs until they are due again. With ``--repair``, an
+        interrupted archive check may leave some archives
         already repaired and others not yet processed, so run ``borg check --repair`` again to
         finish.
 
diff --git a/src/borg/repository.py b/src/borg/repository.py
index b2ccff8157..125ba4b2b2 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1057,9 +1057,8 @@ def recorded_ts(info):
                 pack_infos.sort(key=recorded_ts)
             pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs")
             for info in pack_infos:
-                if sig_int:  # on Ctrl-C, persist checked packs, then stop
-                    logger.info(f"Interrupted repository check, {len(tracker)} packs checked so far.")
-                    tracker.save()
+                if sig_int:  # on Ctrl-C, stop; tracker.prune() below persists the records past the loop
+                    logger.info(f"Interrupted repository check, {pack_files} packs checked so far.")
                     break
                 self._lock_refresh()
                 pack_pi.show(increase=1)  # advance for skipped packs too, so the bar tracks packs/, not work done
@@ -1122,6 +1121,7 @@ def recorded_ts(info):
             logger.error(f"{done} {mode} repository check, errors found{so_far} (repository repair not implemented).")
         else:
             logger.error(f"{done} {mode} repository check, errors found{so_far}.")
+        # True means the checked objects were clean; --repair returns True so the caller proceeds to fix them.
         return not problems or repair
 
     def list(self, limit=None, marker=None):

From 488b411d8abf281744bfb11d2274910241e57eed Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 8 Aug 2026 23:05:32 +0200
Subject: [PATCH 061/144] compress: do not copy chunk data to bytes, use the
 buffer protocol

The chunker hands chunk data to the compressors as a memoryview into its
scan buffer, but LZ4._decide (the default compression) coerced it to
bytes first, copying the full plaintext chunk once per chunk just to get
a char* for the C call. Use a typed memoryview instead, so lz4 reads
straight from the chunker's buffer.

CompressorBase.compress (used by CNONE, i.e. also the fallback for
incompressible chunks) did the same coercion, which is only needed in
legacy mode for the ID/level prefix concatenation - keep it there and
pass the buffer through otherwise. This removed a second full copy for
incompressible chunks.

ObfuscateSize.compress now may receive a memoryview from the inner
compressor, so join instead of concatenating (and skip the copy
completely when the added padding size is 0).

Measured on a 5 GiB borg create (lz4, aes256-ocb, half incompressible /
half compressible input): ~3% faster overall, compression-path time per
stored chunk -23%.
---
 src/borg/compress.pyx | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx
index a0d9361810..367196b373 100644
--- a/src/borg/compress.pyx
+++ b/src/borg/compress.pyx
@@ -146,12 +146,11 @@ cdef class CompressorBase:
 
     def compress(self, meta, data):
         """
-        Compress *data* (bytes) and return compression metadata and compressed bytes.
+        Compress *data* (bytes or memoryview) and return compression metadata and compressed data.
         """
-        if not isinstance(data, bytes):
-            data = bytes(data)  # code below does not work with memoryview
         if self.legacy_mode:
-            return None, bytes((self.ID, self.level)) + data
+            # the ID/level prefix concatenation needs bytes (bytes() is a no-op for bytes input)
+            return None, bytes((self.ID, self.level)) + bytes(data)
         else:
             meta["ctype"] = self.ID
             meta["clevel"] = self.level
@@ -276,12 +275,15 @@ class LZ4(DecidingCompressor):
 
         *lz4_data* is the LZ4 result if *compressor* is LZ4 as well, otherwise it is None.
         """
-        if not isinstance(idata, bytes):
-            idata = bytes(idata)  # code below does not work with memoryview
-        cdef int isize = len(idata)
+        cdef const unsigned char[::1] iview = idata
+        cdef int isize = iview.shape[0]
         cdef int osize
-        cdef char *source = idata
+        cdef const char *source
         cdef char *dest
+        if isize == 0:
+            # empty input cannot shrink (and an empty view has no address to take below)
+            return NONE_COMPRESSOR, (meta, None)
+        source = <const char *> &iview[0]
         osize = LZ4_compressBound(isize)
         buf = get_buffer().get(osize)
         dest = <char *> buf
@@ -630,8 +632,11 @@ class ObfuscateSize(CompressorBase):
         addtl_size = self._obfuscate(compr_size) if meta["type"] == ROBJ_FILE_STREAM else 0
         addtl_size = max(0, addtl_size)  # we can only make it longer, not shorter!
         addtl_size = min(MAX_DATA_SIZE - 1024 - compr_size, addtl_size)  # stay away from MAX_DATA_SIZE
-        trailer = bytes(addtl_size)
-        obfuscated_data = compressed_data + trailer
+        if addtl_size:
+            # join, not +: compressed_data may be a memoryview (CNONE passes the input through)
+            obfuscated_data = b"".join([compressed_data, bytes(addtl_size)])
+        else:
+            obfuscated_data = compressed_data
         meta["csize"] = len(obfuscated_data)  # csize is the overall output size of this "obfuscation compressor"
         meta["olevel"] = self.level  # remember the obfuscation level, useful for repo-compress
         return meta, obfuscated_data  # for borg2 it is enough that we have the payload size in meta["psize"]

From 29132090dabb0602a8dbb1988ea4351d404a993f Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 09:53:05 +0200
Subject: [PATCH 062/144] crypto: AEAD encrypt/decrypt directly into the result
 bytes object

_AEAD_BASE.encrypt and .decrypt used a PyMem_Malloc'd scratch buffer
(with a one-block safety margin) and created the returned bytes object
by slicing it, copying the whole ciphertext / plaintext once more per
chunk before freeing the scratch buffer.

Our AEAD ciphers (AES-OCB, chacha20-poly1305) are padding-free, so the
output size is known exactly up front: allocate the result via
PyBytes_FromStringAndSize(NULL, n) and let OpenSSL write into it
directly - no scratch buffer, no copy, no malloc/free per chunk.

Caveat found by the tests: EVP_EncryptFinal_ex is not output-free for
OCB - OpenSSL buffers the partial final block in EncryptUpdate and
emits it at Final (the total is still exactly the input length). As our
AEAD modes are padding-free, Final can never write more than the space
left in the exact-size result buffer, so it writes directly into it; a
length check afterwards verifies the total.

This helps both directions: borg create saves one ciphertext-sized copy
per chunk, and the read paths (extract, mount, check --verify-data)
save a plaintext-sized copy per chunk. Measured on 5 GiB (lz4,
aes256-ocb): create ~1.5% faster, extract ~6% faster.
---
 src/borg/crypto/low_level.pyx | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx
index 9c4d47b09d..af5c42128e 100644
--- a/src/borg/crypto/low_level.pyx
+++ b/src/borg/crypto/low_level.pyx
@@ -514,10 +514,11 @@ cdef class _AEAD_BASE:
         cdef int rc
 
         try:
-            odata = <unsigned char *>PyMem_Malloc(hlen + self.mac_len +
-                                                  ilen + self.cipher_blk_len)
-            if not odata:
-                raise MemoryError
+            # Our AEAD ciphers (OCB, chacha20-poly1305) are padding-free: the ciphertext is
+            # exactly as long as the plaintext. Thus the result can be allocated up front
+            # and the cipher writes directly into it - no scratch buffer, no copy.
+            ret = PyBytes_FromStringAndSize(NULL, hlen + self.mac_len + ilen)
+            odata = <unsigned char *>PyBytes_AsString(ret)
 
             idata = ro_buffer(data)
             idata_acquired = True
@@ -545,16 +546,19 @@ cdef class _AEAD_BASE:
             if not rc:
                 raise CryptoError('EVP_EncryptUpdate failed')
             offset += olen
+            # Final can emit a buffered partial block (OCB does). Our AEAD modes are
+            # padding-free, so it never writes more than the space left in the
+            # exact-size result buffer.
             if not EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen):
                 raise CryptoError('EVP_EncryptFinal_ex failed')
             offset += olen
             if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_GET_TAG, self.mac_len, odata + hlen):
                 raise CryptoError('EVP_CIPHER_CTX_ctrl GET TAG failed')
+            if offset != hlen + self.mac_len + ilen:
+                raise CryptoError('unexpected ciphertext length')
             self.blocks = block_count
-            return odata[:offset]
+            return ret
         finally:
-            if odata:
-                PyMem_Free(odata)
             if hdata_acquired:
                 PyBuffer_Release(&hdata)
             if idata_acquired:
@@ -591,9 +595,11 @@ cdef class _AEAD_BASE:
         cdef int rc
 
         try:
-            odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
-            if not odata:
-                raise MemoryError
+            # Our AEAD ciphers (OCB, chacha20-poly1305) are padding-free: the plaintext is
+            # exactly as long as the ciphertext. Thus the result can be allocated up front
+            # and the cipher writes directly into it - no scratch buffer, no copy.
+            ret = PyBytes_FromStringAndSize(NULL, ilen - hlen - self.mac_len)
+            odata = <unsigned char *>PyBytes_AsString(ret)
 
             idata = ro_buffer(envelope)
             idata_acquired = True
@@ -619,15 +625,18 @@ cdef class _AEAD_BASE:
             offset += olen
             if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_SET_TAG, self.mac_len, <unsigned char *> idata.buf + hlen):
                 raise CryptoError('EVP_CIPHER_CTX_ctrl SET TAG failed')
+            # Final can emit a buffered partial block (OCB does). Our AEAD modes are
+            # padding-free, so it never writes more than the space left in the
+            # exact-size result buffer.
             if not EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen):
                 # a failure here means corrupted or tampered tag (mac) or data.
                 raise IntegrityError('Authentication / EVP_DecryptFinal_ex failed')
             offset += olen
+            if offset != ilen - hlen - self.mac_len:
+                raise CryptoError('unexpected plaintext length')
             self.blocks = self.block_count(offset)
-            return odata[:offset]
+            return ret
         finally:
-            if odata:
-                PyMem_Free(odata)
             if idata_acquired:
                 PyBuffer_Release(&idata)
             if aadata_acquired:

From 10fbb99bd79b4917ad593d9608ecc859f48103ad Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 11:16:00 +0200
Subject: [PATCH 063/144] compress: lz4 decompresses directly into the result
 bytes object

LZ4.decompress guessed the output size, decompressed into a thread-local
scratch buffer (growing and retrying on overflow) and then copied the
whole plaintext into the returned bytes object - one full plaintext-sized
copy per chunk on every read (extract, mount, check --verify-data).

borg2 stores the exact plaintext size in the (authenticated) object
metadata, so when meta["size"] is known, allocate the result exactly
and let LZ4_decompress_safe write straight into it: no scratch buffer,
no guessing, no copy. A size mismatch raises DecompressionError (this
also covers what check_fix_size asserted).

The guess-and-retry loop remains as the fallback for borg 1.x data read
in legacy mode (borg transfer), where the size is not known up front.

Measured on a 5 GiB extract (lz4, aes256-ocb): ~4% faster.
---
 src/borg/compress.pyx | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx
index 367196b373..57019d6259 100644
--- a/src/borg/compress.pyx
+++ b/src/borg/compress.pyx
@@ -23,6 +23,8 @@ import sys
 import threading
 import zlib
 
+from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AsString
+
 try:
     import lzma
 except ImportError:
@@ -306,6 +308,21 @@ class LZ4(DecidingCompressor):
         cdef int rsize
         cdef char *source = idata
         cdef char *dest
+        size = None if meta is None else meta.get("size")
+        if size is not None and 0 <= size <= 2 ** 31 - 1:
+            # borg2 stores the exact plaintext size in the (authenticated) object metadata:
+            # decompress directly into the result bytes object - no scratch buffer, no copy,
+            # no output size guessing. rsize != size means corrupt (or misdescribed) data,
+            # this also covers everything check_fix_size would assert.
+            ret = PyBytes_FromStringAndSize(NULL, size)
+            dest = PyBytes_AsString(ret)
+            osize = size
+            with nogil:
+                rsize = LZ4_decompress_safe(source, dest, isize, osize)
+            if rsize != osize:
+                raise DecompressionError('lz4 decompress failed')
+            return meta, ret
+        # no size known up front (borg 1.x repos in legacy mode): guess and retry.
         # a bit more than 8MB is enough for the usual data sizes yielded by the chunker.
         # allocate more if isize * 3 is already bigger, to avoid having to resize often.
         osize = max(int(1.1 * 2**23), isize * 3)

From bfc12c0eb197a50ac93f45a74ab9898c574ae08b Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 12:15:17 +0200
Subject: [PATCH 064/144] chunkers: fixed chunker reads directly into per-chunk
 buffers

ChunkerFixed used FileReader.read(), which assembles each chunk in an
intermediate bytearray (copying the data out of the reader's block
buffers) and then converts it to bytes (copying everything again).

Read each chunk via FileReader.readinto() into a fresh per-chunk buffer
instead: each byte is copied exactly once, from the reader's block
buffer into the chunk, and the chunk is yielded as a memoryview over
that buffer. All-zero detection now happens at chunk granularity, like
in the content-defined chunkers.

Also update the reader type stub: FileReader.readinto was missing
there, and Chunk data may be a memoryview (as the CDC chunkers already
yield).

Behavior notes:
- chunk data stays valid after the iterator advances (the buffer is
  per chunk, not reused), matching the previous semantics.
- ranges stemming from holes in sparse files are now reported as
  CH_ALLOC instead of CH_HOLE - downstream treats both identically.

Measured on a 20 GiB create (fixed,4194304, lz4, unencrypted repo,
on top of the copy-removal changes of #10059 which let the yielded
memoryview flow through compression without being copied to bytes):
~10% faster. Standalone (without #10059) only ~2%, as the compressor's
bytes() coercion then re-adds one copy.
---
 src/borg/chunkers/fixed.py   | 43 +++++++++++++++++++++---------------
 src/borg/chunkers/reader.pyi |  5 +++--
 2 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/src/borg/chunkers/fixed.py b/src/borg/chunkers/fixed.py
index 74db7311c2..5db28ed635 100644
--- a/src/borg/chunkers/fixed.py
+++ b/src/borg/chunkers/fixed.py
@@ -4,7 +4,8 @@
 
 import time
 
-from .reader import FileReader
+from .reader import FileReader, Chunk
+from ..constants import CH_DATA, CH_ALLOC, zeros
 
 
 class ChunkerFixed:
@@ -26,6 +27,11 @@ class ChunkerFixed:
 
     Note: the last block of a data or hole range may be less than the block size,
           this is supported and not considered to be an error.
+
+    Chunks are yielded as memoryviews over per-chunk buffers that the file data is
+    read into directly - each byte is copied exactly once, from the reader's block
+    buffer into the chunk (all-zero chunks are yielded as allocation CH_ALLOC with
+    data None). Call release_chunk_data() after consuming a chunk.
     """
 
     def __init__(self, block_size: int, header_size: int = 0, sparse: bool = False) -> None:
@@ -48,24 +54,25 @@ def chunkify(self, fd: BinaryIO | None = None, fh: int = -1, fmap: list | None =
         # Initialize the reader with the file descriptors
         self.reader = FileReader(fd=fd, fh=fh, read_size=self.reader_block_size, sparse=self.sparse, fmap=fmap)
 
-        # Handle header if present
-        if self.header_size > 0:
-            # Read the header block using read
-            started_chunking = time.monotonic()
-            header_chunk = self.reader.read(self.header_size)
-            self.chunking_time += time.monotonic() - started_chunking
-
-            if header_chunk.meta["size"] > 0:
-                # Yield the header chunk
-                yield header_chunk
-
-        # Process the rest of the file using read
+        # Read the header block first, if there is one; after it, read block_size sized blocks.
+        wanted = self.header_size if self.header_size > 0 else self.block_size
         while True:
             started_chunking = time.monotonic()
-            chunk = self.reader.read(self.block_size)
+            # The file data is read directly into a fresh per-chunk buffer (zero-filled
+            # for ranges stemming from holes), so each byte is copied only once, instead
+            # of being assembled in an intermediate buffer and copied out of it again.
+            buf = bytearray(wanted)
+            got = self.reader.readinto(buf, wanted)
             self.chunking_time += time.monotonic() - started_chunking
-            size = chunk.meta["size"]
-            if size == 0:
+            if got == 0:
                 break  # EOF
-            assert size <= self.block_size
-            yield chunk
+            assert got <= wanted
+            data = memoryview(buf)[:got]
+            if zeros.startswith(data):
+                # All-zero chunk: either read as zeros from the file or stemming from
+                # a hole in a sparse file - we can not distinguish that here.
+                data.release()
+                yield Chunk(None, size=got, allocation=CH_ALLOC)
+            else:
+                yield Chunk(data, size=got, allocation=CH_DATA)
+            wanted = self.block_size
diff --git a/src/borg/chunkers/reader.pyi b/src/borg/chunkers/reader.pyi
index b1e6a143c5..2d504a31e6 100644
--- a/src/borg/chunkers/reader.pyi
+++ b/src/borg/chunkers/reader.pyi
@@ -4,10 +4,10 @@ from typing import Any, BinaryIO, NamedTuple
 has_seek_hole: bool
 
 class _Chunk(NamedTuple):
-    data: bytes | None
+    data: bytes | memoryview | None
     meta: dict[str, Any]
 
-def Chunk(data: bytes | None, **meta) -> type[_Chunk]: ...
+def Chunk(data: bytes | memoryview | None, **meta) -> type[_Chunk]: ...
 def release_chunk_data(data: bytes | memoryview | None) -> None: ...
 
 fmap_entry = tuple[int, int, bool]
@@ -39,3 +39,4 @@ class FileReader:
     ) -> None: ...
     def _fill_buffer(self) -> bool: ...
     def read(self, size: int) -> type[_Chunk]: ...
+    def readinto(self, target: bytearray | memoryview, size: int) -> int: ...

From e481b32ca2f1f143ccb6e809ec68df263a30e30d Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 14:56:05 +0200
Subject: [PATCH 065/144] remove unused optr code, fixes #5762

Item.to_optr/from_optr and the whole _item.c (which contained nothing
but _object_to_optr/_optr_to_object) were only used by their own test.

The mechanism - packing a raw PyObject pointer into a bytes object to
pass an object through byte-oriented messaging without serializing it -
was meant for a zeromq-based pipeline that never came to be. Nothing in
borg calls it, and it is unsafe by nature (the docstring itself warned
that the object must not be deallocated between to_optr and from_optr).

Rather than porting it to PyCapsule (as #5762 suggested), remove it:
if a message-passing design ever returns, PyCapsule would be the right
tool to build fresh for it.
---
 src/borg/_item.c                | 41 ---------------------------------
 src/borg/item.pyi               |  3 ---
 src/borg/item.pyx               | 24 -------------------
 src/borg/testsuite/item_test.py |  5 ----
 4 files changed, 73 deletions(-)
 delete mode 100644 src/borg/_item.c

diff --git a/src/borg/_item.c b/src/borg/_item.c
deleted file mode 100644
index c5c78c4074..0000000000
--- a/src/borg/_item.c
+++ /dev/null
@@ -1,41 +0,0 @@
-#include "Python.h"
-
-/*
- * This is not quite as dark magic as it looks. We just convert the address of (pointer to)
- * a PyObject into a bytes object in _wrap_object, and convert these bytes back to the
- * pointer to the original object.
- *
- * This mainly looks a bit confusing due to our mental special-casing of "char*" from other
- * pointers.
- *
- * The big upside to this is that this neither does *any* serialization (beyond creating tiny
- * bytes objects as "stand-ins"), nor has to copy the entire object that's passed around.
- */
-
-static PyObject *
-_object_to_optr(PyObject *obj)
-{
-    /*
-     * Create a temporary reference to the object being passed around so it does not vanish.
-     * Note that we never decref this one in _unwrap_object, since we just transfer that reference
-     * there, i.e. there is an elided "Py_INCREF(x); Py_DECREF(x)".
-     * Since the reference is transferred, calls to _wrap_object and _unwrap_object must be symmetric.
-     */
-    Py_INCREF(obj);
-    return PyBytes_FromStringAndSize((const char*) &obj, sizeof(void*));
-}
-
-static PyObject *
-_optr_to_object(PyObject *bytes)
-{
-    if(!PyBytes_Check(bytes)) {
-        PyErr_SetString(PyExc_TypeError, "Cannot unwrap non-bytes object");
-        return NULL;
-    }
-    if(PyBytes_Size(bytes) != sizeof(void*)) {
-        PyErr_SetString(PyExc_TypeError, "Invalid length of bytes object");
-        return NULL;
-    }
-    PyObject *object = * (PyObject **) PyBytes_AsString(bytes);
-    return object;
-}
diff --git a/src/borg/item.pyi b/src/borg/item.pyi
index 5f63eda377..33baf21665 100644
--- a/src/borg/item.pyi
+++ b/src/borg/item.pyi
@@ -128,9 +128,6 @@ class Item(PropDict):
     def _is_type(self, typetest: Callable) -> bool: ...
     @classmethod
     def create_deleted(self, path) -> Item: ...
-    @classmethod
-    def from_optr(self, optr: Any) -> Item: ...
-    def to_optr(self) -> Any: ...
     @property
     def atime(self) -> int: ...
     @atime.setter
diff --git a/src/borg/item.pyx b/src/borg/item.pyx
index 482a14e001..bec41404ae 100644
--- a/src/borg/item.pyx
+++ b/src/borg/item.pyx
@@ -12,11 +12,6 @@ from .helpers.msgpack import timestamp_to_int, int_to_timestamp, Timestamp
 from .helpers.time import OutputTimestamp, safe_timestamp
 
 
-cdef extern from "_item.c":
-    object _object_to_optr(object obj)
-    object _optr_to_object(object bytes)
-
-
 
 
 
@@ -337,25 +332,6 @@ cdef class Item(PropDict):
                 setattr(self, attr, size)
         return size
 
-    def to_optr(self):
-        """
-        Return an "object pointer" (optr), an opaque bag of bytes.
-        The return value is effectively a reference to this object
-        that can be passed exactly once to Item.from_optr to get this
-        object back.
-
-        to_optr/from_optr must be used symmetrically,
-        don't call from_optr multiple times.
-
-        This object can't be deallocated after a call to to_optr()
-        until from_optr() is called.
-        """
-        return _object_to_optr(self)
-
-    @classmethod
-    def from_optr(self, optr):
-        return _optr_to_object(optr)
-
     @classmethod
     def create_deleted(cls, path):
         return cls(deleted=True, chunks=[], mode=0, path=path)
diff --git a/src/borg/testsuite/item_test.py b/src/borg/testsuite/item_test.py
index 9c6a5b796c..c795430fb3 100644
--- a/src/borg/testsuite/item_test.py
+++ b/src/borg/testsuite/item_test.py
@@ -152,11 +152,6 @@ def test_item_file_size_no_chunks():
     assert item.get_size() == 0
 
 
-def test_item_optr():
-    item = Item()
-    assert Item.from_optr(item.to_optr()) is item
-
-
 @pytest.mark.parametrize(
     "chunk_a, chunk_b, chunks_equal",
     [

From 8998a5a6217177f1f4357d27854cd8d168209145 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 15:30:28 +0200
Subject: [PATCH 066/144] chunkers: read file data directly into the caller's
 buffer

FileReader.readinto copied each byte once: from the 1 MiB blocks that
FileFMAPReader reads (a freshly allocated bytes object per block) into
the caller's buffer.

When there is no sparse processing and no fmap was given - the normal
case - there are no ranges to consider: the file is read start to end.
readinto() then skips the block reader entirely and lets the OS read
directly into the caller's buffer (os.readv, or the file object's own
readinto): zero user-space copies, no per-block allocations, and one
syscall per request (typically a whole chunk or scan-buffer fill)
instead of one per block. fadvise DONTNEED behavior is kept.

This benefits the fixed chunker (which now goes disk -> chunk buffer
with no intermediate steps) as well as the content-defined chunkers,
whose fill() reads into the scan buffer through the same method. The
sparse/fmap path is unchanged.

Measured (20 GiB, unencrypted repo, lz4, page-cache-warm, Apple M3
Pro): fixed,4194304 create ~2-3% faster; for the CDC chunkers the
saved copy is below run-to-run noise. The structural wins: ~20000
fewer 1 MiB allocations and 4x fewer read syscalls per 20 GiB.
---
 src/borg/chunkers/reader.pyi |  1 +
 src/borg/chunkers/reader.pyx | 58 ++++++++++++++++++++++++++++++++----
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/src/borg/chunkers/reader.pyi b/src/borg/chunkers/reader.pyi
index 2d504a31e6..7870952f61 100644
--- a/src/borg/chunkers/reader.pyi
+++ b/src/borg/chunkers/reader.pyi
@@ -38,5 +38,6 @@ class FileReader:
         fmap: list[fmap_entry] = None,
     ) -> None: ...
     def _fill_buffer(self) -> bool: ...
+    def _readinto_direct(self, tv: memoryview, size: int) -> int: ...
     def read(self, size: int) -> type[_Chunk]: ...
     def readinto(self, target: bytearray | memoryview, size: int) -> int: ...
diff --git a/src/borg/chunkers/reader.pyx b/src/borg/chunkers/reader.pyx
index c3ba7c8405..f6866de03f 100644
--- a/src/borg/chunkers/reader.pyx
+++ b/src/borg/chunkers/reader.pyx
@@ -15,6 +15,9 @@ from ..constants import CH_DATA, CH_ALLOC, CH_HOLE, zeros
 # because the FS also needs to support this.
 has_seek_hole = hasattr(os, 'SEEK_DATA') and hasattr(os, 'SEEK_HOLE')
 
+# os.readv is POSIX; on platforms without it (win32) we fall back to os.read + copy.
+has_readv = hasattr(os, 'readv')
+
 _Chunk = namedtuple('_Chunk', 'meta data')
 _Chunk.__doc__ = """\
     Chunk namedtuple
@@ -234,6 +237,12 @@ class FileReader:
         self.fd = fd
         self.fh = fh
         self.fmap = fmap
+        # Without sparse processing and without a given fmap there are no ranges to
+        # consider - the file is read start to end. readinto() then reads directly
+        # from the file into the caller's buffer (see there), instead of going
+        # through the block reader.
+        self.direct = not sparse and fmap is None
+        self.direct_offset = 0  # bytes read so far via the direct path (for fadvise)
 
     def _fill_buffer(self):
         """
@@ -358,22 +367,59 @@ class FileReader:
             # Otherwise, all chunks were CH_ALLOC
             return Chunk(None, size=bytes_read, allocation=CH_ALLOC)
 
+    def _readinto_direct(self, tv, size):
+        """Read up to 'size' bytes from the file directly into 'tv' (a writable memoryview)."""
+        pos = 0
+        while pos < size:
+            if self.fh >= 0:
+                if has_readv:
+                    got = os.readv(self.fh, [tv[pos:size]])
+                else:
+                    data = os.read(self.fh, size - pos)
+                    got = len(data)
+                    tv[pos:pos + got] = data
+                if got > 0:
+                    safe_fadvise(self.fh, self.direct_offset, got, "DONTNEED")
+            else:
+                try:
+                    got = self.fd.readinto(tv[pos:size])
+                except AttributeError:
+                    # file-like object without readinto: fall back to read + copy
+                    data = self.fd.read(size - pos)
+                    got = len(data)
+                    tv[pos:pos + got] = data
+            if not got:
+                break  # EOF
+            pos += got
+            self.direct_offset += got
+        return pos
+
     def readinto(self, target, size):
         """
         Read up to 'size' bytes from the file directly into 'target' (a writable
         buffer, e.g. a memoryview over the caller's scan buffer).
 
-        Unlike read(), this does not allocate or combine intermediate byte
-        objects: each byte is copied exactly once, from the buffered file block
-        into 'target'. Ranges stemming from holes / all-zero blocks are written
-        as zero bytes ('target' may contain stale data). The caller detects
-        all-zero chunks itself at chunk granularity, so no allocation type is
-        returned.
+        Fast path (no sparse processing, no fmap given): the file data is read
+        by the OS directly into 'target' - zero copies in user space and one
+        syscall per request instead of one per block.
+
+        Otherwise, unlike read(), this does not allocate or combine intermediate
+        byte objects: each byte is copied exactly once, from the buffered file
+        block into 'target'. Ranges stemming from holes / all-zero blocks are
+        written as zero bytes ('target' may contain stale data). The caller
+        detects all-zero chunks itself at chunk granularity, so no allocation
+        type is returned.
 
         :param target: writable buffer, len(target) >= size
         :param size: number of bytes to read
         :return: number of bytes written to target (0 at EOF).
         """
+        if self.direct and self.blockify_gen is None:
+            # blockify_gen check: if read() was used on this reader before, keep
+            # using the buffered path, for consistent file position and buffer state.
+            with memoryview(target) as tv:
+                return self._readinto_direct(tv, size)
+
         # Initialize if not already done
         if self.blockify_gen is None:
             self.buffer = []

From 39d0e27f5637fb675a589921f0a3584e61cd030e Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 16:31:23 +0200
Subject: [PATCH 067/144] borg help environment: new help topic about env
 variables, fixes #10061

Add an "environment" help topic (alias: "env") showing the documentation
about the environment variables borg supports, like borg help already does
for patterns, placeholders, compression and match-archives.

The help topic is now the single source of that documentation:
scripts/make.py build_usage generates docs/usage/general/environment.rst.inc
from it (and skips it on the misc. help docs page, where it would duplicate
the "General" chapter's section), and build_man already generates a
borg-environment(1) man page from any help topic.

The text was adapted a bit so that it renders well both via sphinx and via
nanorst on the terminal: the list-table, note directive and hyperlink
targets were replaced by constructs nanorst understands.

nanorst: end a code block when a following line is less indented than the
code block's contents (instead of the hardcoded "less than 4 spaces" rule),
so code blocks nested inside definition lists do not swallow the text
following them any more. Also, borg help <topic> now resolves :ref:
references like the command epilogs already did.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/man_intro.rst                           |   2 +-
 docs/usage/general/environment.rst.inc       | 135 +++---
 scripts/make.py                              |  14 +
 src/borg/archiver/__init__.py                |   1 +
 src/borg/archiver/_common.py                 |   5 +
 src/borg/archiver/help_cmd.py                | 419 ++++++++++++++++++-
 src/borg/helpers/nanorst.py                  |  35 +-
 src/borg/testsuite/archiver/help_cmd_test.py |   6 +-
 src/borg/testsuite/helpers/nanorst_test.py   |  13 +
 9 files changed, 530 insertions(+), 100 deletions(-)

diff --git a/docs/man_intro.rst b/docs/man_intro.rst
index ec3ede1b1d..7054f651b3 100644
--- a/docs/man_intro.rst
+++ b/docs/man_intro.rst
@@ -48,7 +48,7 @@ SEE ALSO
 `borg-delete(1)`, `borg-prune(1)`, `borg-compact(1)`,
 `borg-recreate(1)`
 
-`borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)`
+`borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)`, `borg-environment(1)`
 
 * Main web site https://www.borgbackup.org/
 * Releases https://github.com/borgbackup/borg/releases
diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc
index 49f16d5622..5132f4e53b 100644
--- a/docs/usage/general/environment.rst.inc
+++ b/docs/usage/general/environment.rst.inc
@@ -1,6 +1,8 @@
 Environment Variables
 ~~~~~~~~~~~~~~~~~~~~~
 
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
 Borg uses some environment variables for automation:
 
 General:
@@ -61,7 +63,8 @@ General:
         You can set the default value for the ``--lock-wait`` option with this, so
         you do not need to give it as a command line option.
     BORG_LOGGING_CONF
-        When set, use the given filename as INI_-style logging configuration.
+        When set, use the given filename as INI-style logging configuration (see
+        https://docs.python.org/3/library/logging.config.html#configuration-file-format).
         A basic example conf can be found at ``docs/misc/logging.conf``.
     BORG_RSH
         When set, use this command instead of ``ssh``. This can be used to specify ssh options, such as
@@ -294,60 +297,31 @@ Some automatic "answerers" (if set, they automatically answer confirmation quest
     answer or ask you interactively, depending on whether retries are allowed (they by default are
     allowed). So please test your scripts interactively before making them a non-interactive script.
 
-.. _XDG env var: https://specifications.freedesktop.org/basedir-spec/0.6/ar01s03.html
-
 Directories and files:
-
-    .. note::
-
-        Borg 2 uses the `platformdirs <https://pypi.org/project/platformdirs/>`_ library to determine
-        default directory locations. This means that default paths are **platform-specific**:
-
-        - **Linux**: Uses XDG Base Directory Specification paths (e.g., ``~/.config/borg``,
-          ``~/.cache/borg``, ``~/.local/share/borg``). `XDG env var`_ variables are honoured.
-        - **macOS**: Uses native macOS directories by default (e.g., ``~/Library/Application Support/borg``,
-          ``~/Library/Caches/borg``). `XDG env var`_ variables are honoured if set.
-        - **Windows**: Uses Windows AppData directories (e.g., ``C:\Users\<user>\AppData\Roaming\borg``,
-          ``C:\Users\<user>\AppData\Local\borg``). `XDG env var`_ variables are **not** honoured.
-
-        On all platforms, you can override each directory individually using the specific environment
-        variables described below. You can also set ``BORG_BASE_DIR`` to force borg to use
-        ``BORG_BASE_DIR/.config/borg``, ``BORG_BASE_DIR/.cache/borg``, etc., regardless of the platform.
-
-    Default directory locations by platform (when no ``BORG_*`` environment variables are set):
-
-    .. list-table::
-       :header-rows: 1
-       :widths: 15 25 30 30
-
-       * - Directory
-         - Linux
-         - macOS
-         - Windows
-       * - Config
-         - ``~/.config/borg``
-         - ``~/Library/Application Support/borg``
-         - ``%APPDATA%\borg``
-       * - Cache
-         - ``~/.cache/borg``
-         - ``~/Library/Caches/borg``
-         - ``%LOCALAPPDATA%\borg\Cache``
-       * - Data
-         - ``~/.local/share/borg``
-         - ``~/Library/Application Support/borg``
-         - ``%LOCALAPPDATA%\borg``
-       * - Runtime
-         - ``/run/user/<uid>/borg``
-         - ``~/Library/Caches/TemporaryItems/borg``
-         - ``%TEMP%\borg``
-       * - Keys
-         - ``<config_dir>/keys``
-         - ``<config_dir>/keys``
-         - ``<config_dir>\keys``
-       * - Security
-         - ``<data_dir>/security``
-         - ``<data_dir>/security``
-         - ``<data_dir>\security``
+    Borg 2 uses the platformdirs library (https://pypi.org/project/platformdirs/) to determine
+    default directory locations. This means that default paths are **platform-specific**:
+
+    - Linux: XDG Base Directory Specification paths are used (e.g. ``~/.config/borg``,
+      ``~/.cache/borg``, ``~/.local/share/borg``). ``XDG_*`` environment variables are
+      honoured (see https://specifications.freedesktop.org/basedir-spec/0.6/ar01s03.html).
+    - macOS: native macOS directories are used by default (e.g. ``~/Library/Application Support/borg``,
+      ``~/Library/Caches/borg``). ``XDG_*`` environment variables are honoured if set.
+    - Windows: Windows AppData directories are used (e.g. ``C:\Users\<user>\AppData\Roaming\borg``,
+      ``C:\Users\<user>\AppData\Local\borg``). ``XDG_*`` environment variables are **not** honoured.
+
+    On all platforms, you can override each directory individually using the specific environment
+    variables described below. You can also set ``BORG_BASE_DIR`` to force borg to use
+    ``BORG_BASE_DIR/.config/borg``, ``BORG_BASE_DIR/.cache/borg``, etc., regardless of the platform.
+
+    Default directory locations by platform (when no ``BORG_*`` environment variables are set)::
+
+        Directory  Linux                 macOS                                 Windows
+        Config     ~/.config/borg        ~/Library/Application Support/borg    %APPDATA%\borg
+        Cache      ~/.cache/borg         ~/Library/Caches/borg                 %LOCALAPPDATA%\borg\Cache
+        Data       ~/.local/share/borg   ~/Library/Application Support/borg    %LOCALAPPDATA%\borg
+        Runtime    /run/user/<uid>/borg  ~/Library/Caches/TemporaryItems/borg  %TEMP%\borg
+        Keys       <config_dir>/keys     <config_dir>/keys                     <config_dir>\keys
+        Security   <data_dir>/security   <data_dir>/security                   <data_dir>\security
 
     BORG_BASE_DIR
         Defaults to ``$HOME`` or ``~$USER`` or ``~`` (in that order).
@@ -357,26 +331,26 @@ Directories and files:
     BORG_CACHE_DIR
         Defaults to the platform-specific cache directory (see table above).
         If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.cache/borg``.
-        On Linux and macOS, `XDG env var`_ ``XDG_CACHE_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
+        On Linux and macOS, ``XDG_CACHE_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
         This directory contains the local cache and might need a lot
         of space for dealing with big repositories. Make sure you're aware of the associated
         security aspects of the cache location: :ref:`cache_security`
     BORG_CONFIG_DIR
         Defaults to the platform-specific config directory (see table above).
         If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.config/borg``.
-        On Linux and macOS, `XDG env var`_ ``XDG_CONFIG_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
+        On Linux and macOS, ``XDG_CONFIG_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
         This directory contains all borg configuration directories, see the FAQ
         for a security advisory about the data in this directory: :ref:`home_config_borg`
     BORG_DATA_DIR
         Defaults to the platform-specific data directory (see table above).
         If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.local/share/borg``.
-        On Linux and macOS, `XDG env var`_ ``XDG_DATA_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
+        On Linux and macOS, ``XDG_DATA_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
         This directory contains all borg data directories, see the FAQ
         for a security advisory about the data in this directory: :ref:`home_data_borg`
     BORG_RUNTIME_DIR
         Defaults to the platform-specific runtime directory (see table above).
         If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.cache/borg``.
-        On Linux and macOS, `XDG env var`_ ``XDG_RUNTIME_DIR`` is also honoured if ``BORG_BASE_DIR`` is not set.
+        On Linux and macOS, ``XDG_RUNTIME_DIR`` is also honoured if ``BORG_BASE_DIR`` is not set.
         This directory contains borg runtime files, like e.g. the socket file.
     BORG_SECURITY_DIR
         Defaults to ``$BORG_DATA_DIR/security``.
@@ -397,7 +371,8 @@ Directories and files:
         - you need to point to the correct key file matching the repository the command will operate on.
     TMPDIR
         This is where temporary files are stored (might need a lot of temporary space for some
-        operations), see tempfile_ for details.
+        operations), see https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir
+        for details.
 
 Building:
     BORG_OPENSSL_NAME
@@ -411,38 +386,30 @@ Building:
         Adds given prefix directory to the default locations. If a 'include/lz4.h' is found Borg
         will be linked against the system liblz4 instead of a bundled implementation. (setup.py)
 
-Please note:
-
-- Be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data's security/safety.
-- Also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions (e.g.
-  mode 600, root:root).
+Automatic option environment variables:
+    Borg uses jsonargparse (https://jsonargparse.readthedocs.io/) with ``default_env=True``,
+    which means that every command-line option can also be set via an environment variable.
 
-.. _INI: https://docs.python.org/3/library/logging.config.html#configuration-file-format
+    The environment variable name is derived from the program name (``borg``),
+    the subcommand (if any), and the option name, all converted to uppercase
+    with dashes replaced by underscores.
 
-.. _tempfile: https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir
+    For **top-level options** (not specific to a subcommand), the pattern is::
 
+        BORG_<OPTION>
 
-Automatically generated Environment Variables (jsonargparse)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    For example, ``--lock-wait`` can be set via ``BORG_LOCK_WAIT``.
 
-Borg uses jsonargparse_ with ``default_env=True``, which means that every
-command-line option can also be set via an environment variable.
+    For **subcommand options**, the subcommand and option are separated by a
+    double underscore::
 
-The environment variable name is derived from the program name (``borg``),
-the subcommand (if any), and the option name, all converted to uppercase
-with dashes replaced by underscores.
+        BORG_<SUBCOMMAND>__<OPTION>
 
-For **top-level options** (not specific to a subcommand), the pattern is::
+    For example, ``borg create --comment`` can be set via ``BORG_CREATE__COMMENT``.
 
-    BORG_<OPTION>
-
-For example, ``--lock-wait`` can be set via ``BORG_LOCK_WAIT``.
-
-For **subcommand options**, the subcommand and option are separated by a
-double underscore::
-
-    BORG_<SUBCOMMAND>__<OPTION>
+Please note:
 
-For example, ``borg create --comment`` can be set via ``BORG_CREATE__COMMENT``.
+- Be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data's security/safety.
+- Also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions (e.g.
+  mode 600, root:root).
 
-.. _jsonargparse: https://jsonargparse.readthedocs.io/
diff --git a/scripts/make.py b/scripts/make.py
index 07a32ed653..4023ad0699 100644
--- a/scripts/make.py
+++ b/scripts/make.py
@@ -70,6 +70,10 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
                 doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
                 if command == "help":
                     for topic in Archiver.helptext:
+                        if topic == "environment":
+                            # not on the misc. help docs page - it is included by the
+                            # "General" docs chapter instead, see below.
+                            continue
                         params = {"topic": topic, "underline": "~" * len("borg help " + topic)}
                         doc.write(".. _borg_{topic}:\n\n".format(**params))
                         doc.write("borg help {topic}\n{underline}\n\n".format(**params))
@@ -93,6 +97,16 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
                     doc.write("\n\nDescription\n~~~~~~~~~~~\n")
                     doc.write(epilog)
 
+            if command == "help":
+                # the "environment" help topic is the single source for the
+                # "Environment Variables" section of the "General" docs chapter.
+                # note: the comment must come after the heading - if it came first, it would break
+                # the association of the ``.. _env_vars:`` label (in the including file) with the heading.
+                with open("docs/usage/general/environment.rst.inc", "w") as doc:
+                    doc.write("Environment Variables\n~~~~~~~~~~~~~~~~~~~~~\n\n")
+                    doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n")
+                    doc.write(Archiver.helptext["environment"])
+
         if "create" in choices:
             common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
             with open("docs/usage/common-options.rst.inc", "w") as doc:
diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py
index 8e8c0f40f2..e3a35c99be 100644
--- a/src/borg/archiver/__init__.py
+++ b/src/borg/archiver/__init__.py
@@ -260,6 +260,7 @@ def build_parser(self):
           match-archives      selection patterns for matching archives
           placeholders        placeholders in repository URLs, archive names, etc.
           compression         options and specifications for data compression
+          environment         environment variables used by borg (short: env)
         """
         )
         parser = ArgumentParser(
diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py
index a07cb30bfb..793515450b 100644
--- a/src/borg/archiver/_common.py
+++ b/src/borg/archiver/_common.py
@@ -281,6 +281,11 @@ def wrapper(self, args, repository, manifest, **kwargs):
     "key_files": "Internals -> Data structures and file formats -> Key files",
     "borg_key_export": "borg key export --help",
     "internals_hashindex": "Internals -> Data structures and file formats -> HashIndex",
+    "borg_serve": "borg serve --help",
+    "debugging": '"Debugging Facilities"',
+    "cache_security": 'FAQ -> "Do I need to take security precautions regarding the cache?"',
+    "home_config_borg": 'FAQ -> "How important is the borg config directory?"',
+    "home_data_borg": 'FAQ -> "How important is the borg data directory?"',
 }
 
 
diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py
index 7df5fe6190..296eeb126e 100644
--- a/src/borg/archiver/help_cmd.py
+++ b/src/borg/archiver/help_cmd.py
@@ -1,5 +1,6 @@
 import textwrap
 
+from ._common import rst_plain_text_references
 from ..helpers.argparsing import ArgumentParser
 from ..constants import *  # NOQA
 from ..helpers.nanorst import rst_to_terminal
@@ -569,15 +570,429 @@ class HelpMixIn:
             borg create --compression obfuscate,2,zstd,6 ...
             borg create --compression obfuscate,250,zstd,3 ...\n\n"""
     )
+    helptext["environment"] = textwrap.dedent(
+        """
+        Borg uses some environment variables for automation:
+
+        General:
+            BORG_REPO
+                When set, use the value to give the default repository location.
+                Use this so you do not need to type ``--repo /path/to/my/repo`` all the time.
+            BORG_OTHER_REPO
+                Similar to BORG_REPO, but gives the default for ``--other-repo``.
+            BORG_PASSPHRASE (and BORG_OTHER_PASSPHRASE)
+                When set, use the value to answer the passphrase question for encrypted repositories.
+                It is used when a passphrase is needed to access an encrypted repo as well as when a new
+                passphrase should be initially set when initializing an encrypted repo.
+                See also BORG_NEW_PASSPHRASE.
+            BORG_PASSCOMMAND (and BORG_OTHER_PASSCOMMAND)
+                When set, use the standard output of the command (trailing newlines are stripped) to answer the
+                passphrase question for encrypted repositories.
+                It is used when a passphrase is needed to access an encrypted repo as well as when a new
+                passphrase should be initially set when initializing an encrypted repo. Note that the command
+                is executed without a shell. So variables, like ``$HOME`` will work, but ``~`` won't.
+                If BORG_PASSPHRASE is also set, it takes precedence.
+                See also BORG_NEW_PASSPHRASE.
+            BORG_PASSPHRASE_FD (and BORG_OTHER_PASSPHRASE_FD)
+                When set, specifies a file descriptor to read a passphrase
+                from. Programs starting borg may choose to open an anonymous pipe
+                and use it to pass a passphrase. This is safer than passing via
+                BORG_PASSPHRASE, because on some systems (e.g. Linux) environment
+                can be examined by other processes.
+                If BORG_PASSPHRASE or BORG_PASSCOMMAND are also set, they take precedence.
+            BORG_NEW_PASSPHRASE
+                When set, use the value to answer the passphrase question when a **new** passphrase is asked for.
+                This variable is checked first. If it is not set, BORG_PASSPHRASE and BORG_PASSCOMMAND will also
+                be checked.
+                Main use case for this is to fully automate ``borg key change-passphrase``.
+            BORG_DISPLAY_PASSPHRASE
+                When set, use the value to answer the "display the passphrase for verification" question when defining a new passphrase for encrypted repositories.
+            BORG_DEBUG_PASSPHRASE
+                When set to YES, display debugging information that includes passphrases used and passphrase related env vars set.
+            BORG_EXIT_CODES
+                When set to "modern", the borg process will return more specific exit codes (rc).
+                When set to "legacy", the borg process will return rc 2 for all errors, 1 for all warnings, 0 for success.
+                Default is "modern".
+            BORG_HOST_ID
+                Borg usually computes a host id from the FQDN plus the results of ``uuid.getnode()`` (which usually returns
+                a unique id based on the MAC address of the network interface. Except if that MAC happens to be all-zero - in
+                that case it returns a random value, which is not what we want (because it kills automatic stale lock removal).
+                So, if you have an all-zero MAC address or other reasons to better control the host id externally, just set this
+                environment variable to a unique value. If all your FQDNs are unique, you can just use the FQDN. If not,
+                use FQDN@uniqueid.
+            BORG_HOSTNAME
+                When set, use this value as the hostname (instead of the auto-detected one), e.g. to run borg
+                on one host, but impersonate another host. This affects the hostname stored in newly created
+                archives as well as the ``{hostname}`` placeholder.
+            BORG_USERNAME
+                When set, use this value as the username (instead of the auto-detected one), e.g. to run borg
+                as one user, but impersonate another user. This affects the username stored in newly created
+                archives as well as the ``{user}`` placeholder.
+            BORG_LOCK_WAIT
+                You can set the default value for the ``--lock-wait`` option with this, so
+                you do not need to give it as a command line option.
+            BORG_LOGGING_CONF
+                When set, use the given filename as INI-style logging configuration (see
+                https://docs.python.org/3/library/logging.config.html#configuration-file-format).
+                A basic example conf can be found at ``docs/misc/logging.conf``.
+            BORG_RSH
+                When set, use this command instead of ``ssh``. This can be used to specify ssh options, such as
+                a custom identity file ``ssh -i /path/to/private/key``. See ``man ssh`` for other options.
+                This is the replacement for the removed ``--rsh CMD`` command line option.
+                borg also gives this to borgstore as ``BORGSTORE_RSH``, except if that is already set.
+            BORG_REMOTE_PATH
+                When set, use the given path as borg executable on the remote (defaults to "borg" if unset).
+                This is the replacement for the removed ``--remote-path PATH`` command line option.
+            BORG_UNITS
+                Determines how borg formats sizes in its human-readable output:
+
+                - ``si`` (default): decimal units, e.g. ``1.23 MB`` (1kB = 1000B)
+                - ``iec``: binary units, e.g. ``1.18 MiB`` (1KiB = 1024B)
+                - ``raw``: exact byte counts, e.g. ``1234567 B``
+
+                Use ``raw`` if you want to parse sizes with scripts (e.g. for monitoring),
+                so you do not have to deal with scaled values and different units.
+                Alternatively, use a command's ``--json`` output or, for the commands
+                supporting ``--format``, the size related format keys - sizes are given
+                as byte counts there anyway.
+
+                ``BORG_UNITS=iec`` is the replacement for the removed ``BORG_IEC`` environment
+                variable (and for the ``--iec`` command line option removed before that).
+            BORG_PROGRESS_FPS
+                How often the ``--progress`` output is updated at most, in updates per
+                second (default: 5). Fractional values are allowed, e.g.
+                ``BORG_PROGRESS_FPS=0.1`` limits it to one update every 10 seconds.
+                Lower values are useful when the output goes into a logfile rather than
+                to an interactive terminal.
+            BORG_DEBUG_PROFILE
+                When set to a filename, write an execution profile in Borg format into that file
+                (see :ref:`debugging`). If the filename ends with ``.pyprof``, a Python-compatible
+                profile is written instead.
+                This is the replacement for the removed ``--debug-profile`` command line option.
+                Note: every borg invocation writes the profile, so unset it again when you are done.
+            BORG_REPO_PERMISSIONS
+                Set repository permissions, see also: :ref:`borg_serve`
+            BORG_FILES_CACHE_SUFFIX
+                When set to a value at least one character long, instructs borg to use a specifically named
+                (based on the suffix) alternative files cache. This can be used to avoid loading and saving
+                cache entries for backup sources other than the current sources.
+            BORG_FILES_CACHE_TTL
+                When set to a numeric value, this determines the maximum "time to live" for the files cache
+                entries (default: 2). The files cache is used to determine quickly whether a file is unchanged.
+            BORG_ASSERT_ID
+                Comma-separated list of the places where borg shall verify that a chunk's content matches
+                its chunk id (``chunkid == id_hash(content)``) after decrypting and decompressing it.
+                Verifying costs a full hash pass over everything that is read at such a place.
+
+                Default (variable not set)::
+
+                    BORG_ASSERT_ID=repair,transfer,rechunk
+
+                These are the place names that can be listed:
+
+                read
+                    Every read that decompresses a chunk: ``borg extract``, ``borg mount``,
+                    ``borg export-tar``, ``borg diff``, ... This is by far the most data borg reads, so
+                    this place is **not** in the default, see the explanation below.
+                repair
+                    ``borg check --repair``. It rebuilds archives from the item metadata stream it reads,
+                    re-packing it into new chunks with freshly computed ids, and it recreates manifest and
+                    archives directory entries from what it reads.
+                transfer
+                    ``borg transfer``, for everything it reads from the source repository. Transferring
+                    re-anchors the content in another repository, which is a trust boundary.
+                rechunk
+                    ``borg recreate --chunker-params ...``, i.e. re-chunking reads. Re-chunking computes
+                    new chunk ids from the content it reads, so a violation would not be noticeable any
+                    more afterwards. (Re-chunking in ``borg transfer`` is covered by ``transfer``.)
+
+                An unknown place name is an error. An empty value (``BORG_ASSERT_ID=``) verifies at none of
+                these places, but still where borg always verifies (see below).
+
+                Why ``read`` is not in the default: for encrypted repositories (all the AEAD ciphersuites),
+                the chunk id is part of the AEAD additional authenticated data, so a successful decryption
+                already proves that a holder of the repository key deliberately stored exactly this
+                ciphertext for exactly this chunk id. A malicious or buggy **repository** can therefore not
+                swap, splice or substitute objects, whether the id is verified on read or not. What the id
+                check adds is the detection of chunks whose content does not match their id, which only a
+                malicious or compromised **borg client that had your borg key** could have written (e.g. to
+                poison future deduplication). If that is in your threat model - e.g. because some machines
+                writing into the repository are not fully trusted - add ``read`` to the list::
+
+                    BORG_ASSERT_ID=read,repair,transfer,rechunk
+
+                Otherwise, running ``borg check --verify-data`` periodically is recommended: it is the
+                audit that re-certifies the invariant for all chunks in the background, instead of on
+                every read.
+
+                Independent of this variable, borg always verifies the chunk id:
+
+                - in ``borg check --verify-data``. That audit is what makes not verifying elsewhere
+                  defensible, so it is not configurable (there is no ``verify_data`` place name).
+                - for ``authenticated`` and ``none`` mode repositories: there is no AEAD there, so the id
+                  check *is* the read path's integrity check and switching it off would remove it
+                  completely. Same for reading borg 1.x repositories (``borg transfer``).
+            BORG_BLAKE3_MT_THRESHOLD
+                When set to a numeric value, chunks of at least that many KiB get their id computed by
+                multi-threaded BLAKE3, smaller ones single-threaded (default: 256, i.e. 256KiB).
+                Only relevant for repositories using ``--id-hash blake3``.
+                Multi-threading only pays off for big enough chunks and the break-even point depends on
+                the machine's core count, so the default is deliberately conservative.
+                Run ``scripts/blake3-optimize-mt-threshold.py`` to measure the best value for your
+                machine - it sweeps input sizes, prints the recommended threshold and the command to
+                set it, and can optionally show a chart of the measurements in your browser
+                (``--html --open``).
+                0 means "always multi-threaded", a very large value effectively disables multi-threading.
+            BORG_ZSTD_MT_WORKERS
+                When set to a numeric value, use that many threads to zstd-compress a single chunk
+                (default: the cpu count). 0 or 1 means single-threaded compression.
+                Only relevant when compressing with ``zstd``.
+                Chunks below 768KiB are always compressed single-threaded: libzstd will not use a
+                compression job smaller than 512KiB, so a small chunk gets split very unevenly and
+                multi-threading it would be slower than not doing it at all.
+                Multi-threading trades a little compression ratio for speed (measured at ``zstd,3``:
+                +0.05% archive size for 1MiB chunks, +0.64% for 8MiB ones, more at higher levels), and
+                it uses more cpu time in total to reduce the wallclock time. Set it to 1 if you would
+                rather have the smaller archive, or if borg has to share the cpu with other work.
+            BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL
+                Select the scan kernel the ``fastcdc`` / ``buzhash64`` chunker uses (default:
+                ``scalar``, the plain sequential loop). Accepted values are ``avx512``, ``avx2``,
+                ``neon``, ``blockwise`` and ``scalar``.
+                All kernels chunk identically - same cut points, same chunk ids - and differ only in
+                speed, so this is safe to change at any time, also for an existing repository.
+                Which kernel is fastest is not predictable from the instruction set: it depends on the
+                cpu and on the compiler that built borg, and the sequential loop wins on some machines.
+                Nothing is selected automatically, so measure on your own hardware with
+                ``borg benchmark cpu --chunking`` before setting these.
+                ``avx512`` and ``avx2`` exist only on x86-64, ``neon`` only on aarch64, and only if the
+                compiler that built borg supported them; ``scalar`` and ``blockwise`` are portable C
+                and always available.
+                Requesting a kernel that this build or this cpu cannot run is an error rather than a
+                silent fallback, so a benchmark can not accidentally measure a different kernel.
+                ``borg create --debug`` logs the chunker and the kernel it was created with.
+            BORG_AES_CHUNKER_KERNEL
+                Select the scan kernel used by the AES based chunkers - one variable for all three of
+                ``toeplitz-aes``, ``rabin-aes`` and ``goldilocks-aes`` (default: ``evp``, the portable
+                OpenSSL path). Accepted values are ``vaes``, ``aes-ni``, ``aes-arm64`` and ``evp``.
+                As with the chunker kernels above, all of them chunk identically and differ only in
+                speed, nothing is selected automatically, and a kernel that can not run here is an
+                error rather than a silent fallback.
+                ``vaes`` and ``aes-ni`` exist only on x86-64, ``aes-arm64`` only on aarch64.
+                ``vaes`` additionally needs a compiler that knows it (gcc >= 11 / clang >= 14), so a
+                cpu supporting VAES is not by itself enough to have that kernel available.
+            BORG_SHOW_SYSINFO
+                When set to no (default: yes), system information (like OS, Python version, ...) in
+                exceptions is not shown.
+                Please only use for good reasons as it makes issues harder to analyze.
+            BORG_MSGPACK_VERSION_CHECK
+                Controls whether Borg checks the ``msgpack`` version.
+                The default is ``yes`` (strict check). Set to ``no`` to disable the version check and
+                allow any installed ``msgpack`` version. Use this at your own risk; malfunctioning or
+                incompatible ``msgpack`` versions may cause subtle bugs or repository data corruption.
+            BORG_FUSE_IMPL
+                Choose the low-level FUSE implementation borg shall use for ``borg mount``.
+                This is a comma-separated list of implementation names, they are tried in the
+                given order, e.g.:
+
+                - ``mfusepy,pyfuse3,llfuse``: default, first try to load mfusepy, then pyfuse3, then llfuse.
+                - ``llfuse,pyfuse3``: first try to load llfuse, then try to load pyfuse3.
+                - ``mfusepy``: only try to load mfusepy
+                - ``pyfuse3``: only try to load pyfuse3
+                - ``llfuse``: only try to load llfuse
+                - ``none``: do not try to load an implementation
+            BORG_SELFTEST
+                This can be used to influence borg's built-in self-tests. The default is to execute the tests
+                at the beginning of each borg command invocation.
+
+                BORG_SELFTEST=disabled can be used to switch off the tests and rather save some time.
+                Disabling is not recommended for normal borg users, but large scale borg storage providers can
+                use this to optimize production servers after at least doing a one-time test borg (with
+                self-tests not disabled) when installing or upgrading machines/OS/Borg.
+            BORG_WORKAROUNDS
+                A list of comma-separated strings that trigger workarounds in borg,
+                e.g. to work around bugs in other software.
+
+                Currently known strings are:
+
+                basesyncfile
+                    Use the more simple BaseSyncFile code to avoid issues with sync_file_range.
+                    You might need this to run borg on WSL (Windows Subsystem for Linux) or
+                    in systemd.nspawn containers on some architectures (e.g. ARM).
+                    Using this does not affect data safety, but might result in a more bursty
+                    write-to-disk behavior (not continuously streaming to disk).
+
+                retry_erofs
+                    Retry opening a file without O_NOATIME if opening a file with O_NOATIME
+                    caused EROFS. You will need this to make archives from volume shadow copies
+                    in WSL1 (Windows Subsystem for Linux 1).
+
+                authenticated_no_key
+                    Work around a lost passphrase or key for an ``authenticated`` mode repository
+                    (these are only authenticated, but not encrypted).
+                    If the key is missing in the repository config, add ``key = anything`` there.
+
+                    This workaround is **only** for emergencies and **only** to extract data
+                    from an affected repository (read-only access)::
+
+                        BORG_WORKAROUNDS=authenticated_no_key borg extract --repo repo archive
+
+                    After you have extracted all data you need, you MUST delete the repository::
+
+                        BORG_WORKAROUNDS=authenticated_no_key borg delete repo
+
+                    Now you can init a fresh repo. Make sure you do not use the workaround any more.
+
+        Output formatting:
+            BORG_CHECK_FORMAT
+                Giving the default value for ``borg check --format=X``.
+            BORG_LIST_FORMAT
+                Giving the default value for ``borg list --format=X``.
+            BORG_REPO_LIST_FORMAT
+                Giving the default value for ``borg repo-list --format=X``.
+            BORG_PRUNE_FORMAT
+                Giving the default value for ``borg prune --format=X``.
+
+        Some automatic "answerers" (if set, they automatically answer confirmation questions):
+            BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes)
+                For "Warning: Attempting to access a previously unknown unencrypted repository"
+            BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes)
+                For "Warning: The repository at location ... was previously located at ..."
+            BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
+                For "This is a potentially dangerous function..." (check --repair)
+            BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
+                For "You requested to DELETE the repository completely *including* all archives it contains:"
+
+            Note: answers are case sensitive. setting an invalid answer value might either give the default
+            answer or ask you interactively, depending on whether retries are allowed (they by default are
+            allowed). So please test your scripts interactively before making them a non-interactive script.
+
+        Directories and files:
+            Borg 2 uses the platformdirs library (https://pypi.org/project/platformdirs/) to determine
+            default directory locations. This means that default paths are **platform-specific**:
+
+            - Linux: XDG Base Directory Specification paths are used (e.g. ``~/.config/borg``,
+              ``~/.cache/borg``, ``~/.local/share/borg``). ``XDG_*`` environment variables are
+              honoured (see https://specifications.freedesktop.org/basedir-spec/0.6/ar01s03.html).
+            - macOS: native macOS directories are used by default (e.g. ``~/Library/Application Support/borg``,
+              ``~/Library/Caches/borg``). ``XDG_*`` environment variables are honoured if set.
+            - Windows: Windows AppData directories are used (e.g. ``C:\\Users\\<user>\\AppData\\Roaming\\borg``,
+              ``C:\\Users\\<user>\\AppData\\Local\\borg``). ``XDG_*`` environment variables are **not** honoured.
+
+            On all platforms, you can override each directory individually using the specific environment
+            variables described below. You can also set ``BORG_BASE_DIR`` to force borg to use
+            ``BORG_BASE_DIR/.config/borg``, ``BORG_BASE_DIR/.cache/borg``, etc., regardless of the platform.
+
+            Default directory locations by platform (when no ``BORG_*`` environment variables are set)::
+
+                Directory  Linux                 macOS                                 Windows
+                Config     ~/.config/borg        ~/Library/Application Support/borg    %APPDATA%\\borg
+                Cache      ~/.cache/borg         ~/Library/Caches/borg                 %LOCALAPPDATA%\\borg\\Cache
+                Data       ~/.local/share/borg   ~/Library/Application Support/borg    %LOCALAPPDATA%\\borg
+                Runtime    /run/user/<uid>/borg  ~/Library/Caches/TemporaryItems/borg  %TEMP%\\borg
+                Keys       <config_dir>/keys     <config_dir>/keys                     <config_dir>\\keys
+                Security   <data_dir>/security   <data_dir>/security                   <data_dir>\\security
+
+            BORG_BASE_DIR
+                Defaults to ``$HOME`` or ``~$USER`` or ``~`` (in that order).
+                If you want to move all borg-specific folders to a custom path at once, all you need to do is
+                to modify ``BORG_BASE_DIR``: the other paths for cache, config etc. will adapt accordingly
+                (assuming you didn't set them to a different custom value).
+            BORG_CACHE_DIR
+                Defaults to the platform-specific cache directory (see table above).
+                If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.cache/borg``.
+                On Linux and macOS, ``XDG_CACHE_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
+                This directory contains the local cache and might need a lot
+                of space for dealing with big repositories. Make sure you're aware of the associated
+                security aspects of the cache location: :ref:`cache_security`
+            BORG_CONFIG_DIR
+                Defaults to the platform-specific config directory (see table above).
+                If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.config/borg``.
+                On Linux and macOS, ``XDG_CONFIG_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
+                This directory contains all borg configuration directories, see the FAQ
+                for a security advisory about the data in this directory: :ref:`home_config_borg`
+            BORG_DATA_DIR
+                Defaults to the platform-specific data directory (see table above).
+                If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.local/share/borg``.
+                On Linux and macOS, ``XDG_DATA_HOME`` is also honoured if ``BORG_BASE_DIR`` is not set.
+                This directory contains all borg data directories, see the FAQ
+                for a security advisory about the data in this directory: :ref:`home_data_borg`
+            BORG_RUNTIME_DIR
+                Defaults to the platform-specific runtime directory (see table above).
+                If ``BORG_BASE_DIR`` is set, defaults to ``$BORG_BASE_DIR/.cache/borg``.
+                On Linux and macOS, ``XDG_RUNTIME_DIR`` is also honoured if ``BORG_BASE_DIR`` is not set.
+                This directory contains borg runtime files, like e.g. the socket file.
+            BORG_SECURITY_DIR
+                Defaults to ``$BORG_DATA_DIR/security``.
+                This directory contains security relevant data.
+            BORG_KEYS_DIR
+                Defaults to ``$BORG_CONFIG_DIR/keys``.
+                This directory contains keys for encrypted repositories.
+            BORG_KEY_FILE
+                When set, use the given path as repository key file. Please note that this is only
+                for rather special applications that externally fully manage the key files:
+
+                - this setting only applies to the keyfile modes (not to the repokey modes).
+                - using a full, absolute path to the key file is recommended.
+                - all directories in the given path must exist.
+                - this setting forces borg to use the key file at the given location.
+                - the key file must either exist (for most commands) or will be created (``borg repo-create``).
+                - you need to give a different path for different repositories.
+                - you need to point to the correct key file matching the repository the command will operate on.
+            TMPDIR
+                This is where temporary files are stored (might need a lot of temporary space for some
+                operations), see https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir
+                for details.
+
+        Building:
+            BORG_OPENSSL_NAME
+                Defines the subdirectory name for OpenSSL (setup.py).
+            BORG_OPENSSL_PREFIX
+                Adds given OpenSSL header file directory to the default locations (setup.py).
+            BORG_LIBACL_PREFIX
+                Adds given prefix directory to the default locations. If an 'include/acl/libacl.h' is found
+                Borg will be linked against the system libacl instead of a bundled implementation. (setup.py)
+            BORG_LIBLZ4_PREFIX
+                Adds given prefix directory to the default locations. If a 'include/lz4.h' is found Borg
+                will be linked against the system liblz4 instead of a bundled implementation. (setup.py)
+
+        Automatic option environment variables:
+            Borg uses jsonargparse (https://jsonargparse.readthedocs.io/) with ``default_env=True``,
+            which means that every command-line option can also be set via an environment variable.
+
+            The environment variable name is derived from the program name (``borg``),
+            the subcommand (if any), and the option name, all converted to uppercase
+            with dashes replaced by underscores.
+
+            For **top-level options** (not specific to a subcommand), the pattern is::
+
+                BORG_<OPTION>
+
+            For example, ``--lock-wait`` can be set via ``BORG_LOCK_WAIT``.
+
+            For **subcommand options**, the subcommand and option are separated by a
+            double underscore::
+
+                BORG_<SUBCOMMAND>__<OPTION>
+
+            For example, ``borg create --comment`` can be set via ``BORG_CREATE__COMMENT``.
+
+        Please note:
+
+        - Be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data's security/safety.
+        - Also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions (e.g.
+          mode 600, root:root).\n\n"""
+    )
+    helptext_aliases = {"env": "environment"}
 
     def do_help(self, parser, args):
         commands = getattr(parser, "_subcommands_action", None)
         commands = commands._name_parser_map if commands else {}
+        topic = self.helptext_aliases.get(args.topic, args.topic)
 
         if not args.topic:
             parser.print_help()
-        elif args.topic in self.helptext:
-            print(rst_to_terminal(self.helptext[args.topic]))
+        elif topic in self.helptext:
+            print(rst_to_terminal(self.helptext[topic], rst_plain_text_references))
         elif args.topic in commands:
             if args.epilog_only:
                 print(commands[args.topic].epilog)
diff --git a/src/borg/helpers/nanorst.py b/src/borg/helpers/nanorst.py
index ea3ac20241..e04ecab16c 100644
--- a/src/borg/helpers/nanorst.py
+++ b/src/borg/helpers/nanorst.py
@@ -19,9 +19,9 @@ def peek(self, n):
         else:
             return self.str[self.i + n - 1 : self.i - 1]
 
-    def peekline(self):
+    def peekline(self, offset=0):
         out = ""
-        i = self.i
+        i = self.i + offset
         while i < len(self.str) and self.str[i] != "\n":
             out += self.str[i]
             i += 1
@@ -60,6 +60,7 @@ def rst_to_text(text, state_hook=None, references=None):
     references = references or {}
     state = "text"
     inline_mode = "replace"
+    code_indent = None  # indentation of the current code block's contents
     text = TextPecker(text)
     out = io.StringIO()
 
@@ -111,6 +112,7 @@ def rst_to_text(text, state_hook=None, references=None):
                     text.read(2)
                     state_hook(state, "code-block", out)
                     state = "code-block"
+                    code_indent = None
                     out.write(":\n")
                     continue
             if text.peek(-2) in ("\n\n", "") and char == next == ".":
@@ -144,16 +146,25 @@ def rst_to_text(text, state_hook=None, references=None):
             state = "text"
             text.read(1)
             continue
-        if state == "code-block" and char == next == "\n" and text.peek(5)[1:] != "    ":
-            # Foo::
-            #
-            #     *stuff* *code* *ignore .. all markup*
-            #
-            #     More arcane stuff
-            #
-            # Regular text...
-            state_hook(state, "text", out)
-            state = "text"
+        if state == "code-block":
+            if code_indent is None and char == "\n":
+                line = text.peekline()
+                if line.strip():
+                    code_indent = len(line) - len(line.lstrip())
+            if char == next == "\n":
+                # a blank line ends the code block if the following line is less
+                # indented than the code block's contents:
+                # Foo::
+                #
+                #     *stuff* *code* *ignore .. all markup*
+                #
+                #     More arcane stuff
+                #
+                # Regular text...
+                following = text.peekline(1)
+                if following.strip() and len(following) - len(following.lstrip()) < (code_indent or 1):
+                    state_hook(state, "text", out)
+                    state = "text"
         out.write(char)
 
     if state == "code-block":
diff --git a/src/borg/testsuite/archiver/help_cmd_test.py b/src/borg/testsuite/archiver/help_cmd_test.py
index e411244a9e..5bba6123e5 100644
--- a/src/borg/testsuite/archiver/help_cmd_test.py
+++ b/src/borg/testsuite/archiver/help_cmd_test.py
@@ -1,5 +1,6 @@
 import pytest
 
+from ...archiver._common import rst_plain_text_references
 from ...constants import *  # NOQA
 from ...helpers.nanorst import RstToTextLazy, rst_to_terminal
 from . import Archiver, cmd, exec_cmd
@@ -40,6 +41,8 @@ def test_usage(archiver):
 def test_help(archiver):
     assert "Borg" in cmd(archiver, "help")
     assert "patterns" in cmd(archiver, "help", "patterns")
+    assert "BORG_PASSPHRASE" in cmd(archiver, "help", "environment")
+    assert "BORG_PASSPHRASE" in cmd(archiver, "help", "env")  # alias
     assert "creates a new, empty repository" in cmd(archiver, "help", "repo-create")
     assert "positional arguments" not in cmd(archiver, "help", "repo-create", "--epilog-only")
     assert "creates a new, empty repository" not in cmd(archiver, "help", "repo-create", "--usage-only")
@@ -54,7 +57,7 @@ def test_help_formatting(command, parser):
 @pytest.mark.parametrize("topic", list(Archiver.helptext.keys()))
 def test_help_formatting_helptexts(topic):
     helptext = Archiver.helptext[topic]
-    assert str(rst_to_terminal(helptext))
+    assert str(rst_to_terminal(helptext, rst_plain_text_references))
 
 
 def test_main_help_epilog(archiver):
@@ -64,6 +67,7 @@ def test_main_help_epilog(archiver):
     assert "match-archives" in help_output
     assert "placeholders" in help_output
     assert "compression" in help_output
+    assert "environment" in help_output
 
 
 @pytest.mark.parametrize("command", list(get_all_parsers().keys()))
diff --git a/src/borg/testsuite/helpers/nanorst_test.py b/src/borg/testsuite/helpers/nanorst_test.py
index b47db6c4aa..1bbc61eb53 100644
--- a/src/borg/testsuite/helpers/nanorst_test.py
+++ b/src/borg/testsuite/helpers/nanorst_test.py
@@ -42,3 +42,16 @@ def test_code_block_end_of_string():
     # check that there is no unexpected exception if a code block
     # is not followed by blank lines, but the string just ends.
     assert rst_to_text("This is a code block::\n\n    borg --help") == "This is a code block:\n\n    borg --help"
+
+
+def test_code_block_indented_context():
+    # a code block inside a definition list: the following text is indented, but less than
+    # the code block's contents, so it ends the code block and gets markup-processed again.
+    rst = "term\n    Foo::\n\n        code\n\n    Continued ``text``.\n"
+    assert rst_to_text(rst) == "term\n    Foo:\n\n        code\n\n    Continued text.\n"
+
+
+def test_code_block_less_indented_than_four():
+    # code blocks indented by less than 4 spaces are recognized, too
+    rst = "Foo::\n\n  code\n\nBar ``baz``.\n"
+    assert rst_to_text(rst) == "Foo:\n\n  code\n\nBar baz.\n"

From 4bad6203629580144ff63243dd5ece7afbbe4e58 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 17:37:03 +0200
Subject: [PATCH 068/144] CI: give the test VMs 4 cpus / 8 GiB RAM

The cross-platform-actions default VM size is 2 cpus, which leaves half
of the ubuntu-24.04 runner's 4 vcpus idle and limits pytest-xdist to 2
workers. Use all 4 cpus and 8 GiB of the runner's 16 GiB RAM, roughly
halving the test suite wall clock time on all VM-tested OSes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7f20f77cc7..3926d710dc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -439,6 +439,9 @@ jobs:
           operating_system: ${{ matrix.os }}
           version: ${{ matrix.version }}
           shell: bash
+          # the default VM size (2 cpus) leaves half of the runner's 4 vcpus idle
+          cpu_count: 4
+          memory: 8G
 
       - name: Test on ${{ matrix.display_name }}
         shell: cpa.sh {0}

From a14622d9945d470df2b20a0bb375dda0aba34251 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 17:37:36 +0200
Subject: [PATCH 069/144] CI: time out the "Start VM" step after 15 minutes

The NetBSD VM sometimes never becomes ssh-reachable and
cross-platform-actions retries "Waiting for VM to be ready..." forever,
burning the full 180 minute job timeout (seen twice on master within
one week). A normal boot takes well under a minute, so fail the step
after 15 minutes instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3926d710dc..8cd31077cf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -434,6 +434,9 @@ jobs:
 
       - name: Start VM for ${{ matrix.display_name }}
         id: cross_os
+        # a normal boot takes < 1 minute; sometimes the VM never becomes
+        # ssh-reachable and the action retries forever - fail early then.
+        timeout-minutes: 15
         uses: cross-platform-actions/action@v1.3.0
         with:
           operating_system: ${{ matrix.os }}

From f9e6756d69d752aeeb7f847f9a2fe4f8efa591c9 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 17:38:27 +0200
Subject: [PATCH 070/144] CI: fail hung tests after 5 minutes on the test VMs

test_create_read_special_symlink was seen hanging forever on NetBSD,
stalling one pytest-xdist worker until the 180 minute job timeout.
Install pytest-timeout and set PYTEST_TIMEOUT=300 for the VM test runs,
so a hung test fails with a stack dump instead of hanging the job.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml            | 4 ++++
 requirements.d/development.lock.txt | 1 +
 requirements.d/development.txt      | 1 +
 3 files changed, 6 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8cd31077cf..f473fef4e6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -453,6 +453,10 @@ jobs:
         run: |
           set -euxo pipefail
 
+          # a hung test must fail instead of stalling the suite until the
+          # job timeout (tox passes all env vars through to pytest).
+          export PYTEST_TIMEOUT=300
+
           case "${{ matrix.os }}" in
             freebsd)
               export IGNORE_OSVERSION=yes
diff --git a/requirements.d/development.lock.txt b/requirements.d/development.lock.txt
index 71eb4566a5..0e9a661c5f 100644
--- a/requirements.d/development.lock.txt
+++ b/requirements.d/development.lock.txt
@@ -7,5 +7,6 @@ pytest-xdist==3.8.0
 coverage[toml]==7.15.0
 pytest-cov==7.1.0
 pytest-benchmark==5.2.3
+pytest-timeout==2.4.0
 pre-commit==4.6.1
 types-PyYAML==6.0.12.20260724
diff --git a/requirements.d/development.txt b/requirements.d/development.txt
index 989bc24339..9f8a0c5265 100644
--- a/requirements.d/development.txt
+++ b/requirements.d/development.txt
@@ -7,6 +7,7 @@ pytest-xdist
 coverage[toml]
 pytest-cov
 pytest-benchmark
+pytest-timeout
 pre-commit
 bandit[toml]
 types-PyYAML

From d38a3cc5f4742c3bcd831a1299c8416c00b5be0e Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 18:43:47 +0200
Subject: [PATCH 071/144] tests: benchmark crud json-lines: io throughput may
 round to 0

io is int(bytes/second); with the test's 1-byte samples it is 0
whenever an operation takes >= 1 second, which happens on slow, loaded
CI VMs (seen on OpenBSD with 4 pytest-xdist workers).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/testsuite/archiver/benchmark_cmd_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/borg/testsuite/archiver/benchmark_cmd_test.py b/src/borg/testsuite/archiver/benchmark_cmd_test.py
index f4c16185c9..41399f656c 100644
--- a/src/borg/testsuite/archiver/benchmark_cmd_test.py
+++ b/src/borg/testsuite/archiver/benchmark_cmd_test.py
@@ -44,7 +44,9 @@ def test_benchmark_crud_json_lines(archiver, monkeypatch):
         assert isinstance(entry["time"], float)
         assert entry["time"] > 0
         assert isinstance(entry["io"], int)
-        assert entry["io"] > 0
+        # io is int(bytes/second); with this test's 1-byte samples it is 0
+        # whenever the operation takes >= 1s (seen on a loaded OpenBSD CI VM).
+        assert entry["io"] >= 0
 
 
 def test_benchmark_cpu(archiver, monkeypatch):

From 800a4b8e57ccceb386d4a49d6e249d8b36f81441 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 9 Aug 2026 18:35:04 +0200
Subject: [PATCH 072/144] chunkers: only use the direct read path for known
 regular files

The direct read path (big readv directly into the caller's buffer) was
seen blocking forever on NetBSD 10 when reading a FIFO via
create --read-special (test_create_read_special_symlink hung NetBSD CI
deterministically, at the same spot in two runs), while the buffered
block reader path's 1 MiB os.read() calls work fine there.

Restrict the direct path to known regular files: FileReader only goes
direct for S_ISREG modes. chunkify() accepts the file's os.stat_result
if the caller has it anyway (process_file does, so the create hot path
adds no stat syscall); otherwise FileReader fstats fh / fd.fileno()
itself, staying on the buffered path for objects without an OS-level
fd. Special files (--read-special) route onto the buffered path via
their stat mode, exactly as before this branch.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/archive.py                        |  8 ++-
 src/borg/chunkers/base.pyx                 |  8 ++-
 src/borg/chunkers/buzhash.pyi              |  5 +-
 src/borg/chunkers/buzhash64.pyi            |  5 +-
 src/borg/chunkers/failing.py               |  6 +-
 src/borg/chunkers/fastcdc.pyi              |  5 +-
 src/borg/chunkers/fixed.py                 |  9 ++-
 src/borg/chunkers/goldilocks_aes.pyi       |  5 +-
 src/borg/chunkers/rabin_aes.pyi            |  5 +-
 src/borg/chunkers/reader.pyi               |  2 +
 src/borg/chunkers/reader.pyx               | 26 +++++++--
 src/borg/chunkers/toeplitz_aes.pyi         |  5 +-
 src/borg/testsuite/chunkers/reader_test.py | 66 ++++++++++++++++++++++
 13 files changed, 138 insertions(+), 17 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 271118ac86..727b6e5cac 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -1593,7 +1593,13 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal,
                         # stored. An unwrapped repository OSError is critical and aborts create before
                         # archive.save() runs (see the BackupOSError docstring).
                         self.process_file_chunks(
-                            item, cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(None, fd))
+                            item,
+                            cache,
+                            self.stats,
+                            self.show_progress,
+                            # passing st saves FileReader a stat call; regular files take the
+                            # direct read path, special files (--read-special) the buffered one.
+                            backup_io_iter(self.chunker.chunkify(None, fd, st=st)),
                         )
                         self.stats.chunking_time = self.chunker.chunking_time
                         end_reading = time.time_ns()
diff --git a/src/borg/chunkers/base.pyx b/src/borg/chunkers/base.pyx
index d969adb624..89ae15cf7a 100644
--- a/src/borg/chunkers/base.pyx
+++ b/src/borg/chunkers/base.pyx
@@ -228,7 +228,7 @@ cdef class ChunkerBase:
         self.bytes_yielded += n
         return memoryview((self.data + old_last)[:n])
 
-    def chunkify(self, fd, fh=-1, fmap=None):
+    def chunkify(self, fd, fh=-1, fmap=None, st=None):
         """
         Cut a file into chunks.
 
@@ -236,10 +236,14 @@ cdef class ChunkerBase:
         :param fh: OS-level file handle (if available),
                    defaults to -1 which means not to use OS-level fd.
         :param fmap: a file map, same format as generated by sparsemap
+        :param st: the file's os.stat_result, if the caller has it anyway -
+                   saves FileReader a stat call, see there.
         """
         self._fd = fd
         self.fh = fh
-        self.file_reader = FileReader(fd=fd, fh=fh, read_size=self.reader_block_size, sparse=self.sparse, fmap=fmap)
+        self.file_reader = FileReader(
+            fd=fd, fh=fh, read_size=self.reader_block_size, sparse=self.sparse, fmap=fmap, st=st
+        )
         self.done = 0
         self.remaining = 0
         self.bytes_read = 0
diff --git a/src/borg/chunkers/buzhash.pyi b/src/borg/chunkers/buzhash.pyi
index e8f1ae0670..f910d17317 100644
--- a/src/borg/chunkers/buzhash.pyi
+++ b/src/borg/chunkers/buzhash.pyi
@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import BinaryIO
 
@@ -16,4 +17,6 @@ class Chunker:
         hash_window_size: int,
         sparse: bool = False,
     ) -> None: ...
-    def chunkify(self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None) -> Iterator: ...
+    def chunkify(
+        self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None, st: os.stat_result = None
+    ) -> Iterator: ...
diff --git a/src/borg/chunkers/buzhash64.pyi b/src/borg/chunkers/buzhash64.pyi
index aa40dc1d5d..7cd9f4b7b9 100644
--- a/src/borg/chunkers/buzhash64.pyi
+++ b/src/borg/chunkers/buzhash64.pyi
@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import BinaryIO
 
@@ -19,4 +20,6 @@ class ChunkerBuzHash64:
         normal_size: int = 0,
         sparse: bool = False,
     ) -> None: ...
-    def chunkify(self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None) -> Iterator: ...
+    def chunkify(
+        self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None, st: os.stat_result = None
+    ) -> Iterator: ...
diff --git a/src/borg/chunkers/failing.py b/src/borg/chunkers/failing.py
index 5dc1755476..86a8292837 100644
--- a/src/borg/chunkers/failing.py
+++ b/src/borg/chunkers/failing.py
@@ -27,13 +27,17 @@ def __init__(self, block_size: int, map: str) -> None:
         self.count = 0
         self.chunking_time = 0.0  # not updated, just provided so that caller does not crash
 
-    def chunkify(self, fd: BinaryIO | None = None, fh: int = -1) -> Iterator:
+    def chunkify(
+        self, fd: BinaryIO | None = None, fh: int = -1, fmap: list | None = None, st: os.stat_result | None = None
+    ) -> Iterator:
         """
         Cut a file into chunks.
 
         :param fd: Python file object
         :param fh: OS-level file handle (if available),
                    defaults to -1 which means not to use OS-level fd.
+        :param fmap: ignored
+        :param st: ignored
         """
         use_fh = fh >= 0
         wanted = self.block_size
diff --git a/src/borg/chunkers/fastcdc.pyi b/src/borg/chunkers/fastcdc.pyi
index b9fb63d0e1..085489b28c 100644
--- a/src/borg/chunkers/fastcdc.pyi
+++ b/src/borg/chunkers/fastcdc.pyi
@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import BinaryIO
 
@@ -16,4 +17,6 @@ class ChunkerFastCDC:
         normal_size: int = 0,
         sparse: bool = False,
     ) -> None: ...
-    def chunkify(self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None) -> Iterator: ...
+    def chunkify(
+        self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None, st: os.stat_result = None
+    ) -> Iterator: ...
diff --git a/src/borg/chunkers/fixed.py b/src/borg/chunkers/fixed.py
index 5db28ed635..944ca3d825 100644
--- a/src/borg/chunkers/fixed.py
+++ b/src/borg/chunkers/fixed.py
@@ -2,6 +2,7 @@
 from typing import BinaryIO
 
 
+import os
 import time
 
 from .reader import FileReader, Chunk
@@ -42,7 +43,9 @@ def __init__(self, block_size: int, header_size: int = 0, sparse: bool = False)
         self.reader: FileReader | None = None
         self.sparse = sparse
 
-    def chunkify(self, fd: BinaryIO | None = None, fh: int = -1, fmap: list | None = None) -> Iterator:
+    def chunkify(
+        self, fd: BinaryIO | None = None, fh: int = -1, fmap: list | None = None, st: os.stat_result | None = None
+    ) -> Iterator:
         """
         Cut a file into chunks.
 
@@ -50,9 +53,11 @@ def chunkify(self, fd: BinaryIO | None = None, fh: int = -1, fmap: list | None =
         :param fh: OS-level file handle (if available),
                    defaults to -1 which means not to use OS-level fd.
         :param fmap: a file map, same format as generated by sparsemap
+        :param st: the file's os.stat_result, if the caller has it anyway -
+                   saves FileReader a stat call, see there.
         """
         # Initialize the reader with the file descriptors
-        self.reader = FileReader(fd=fd, fh=fh, read_size=self.reader_block_size, sparse=self.sparse, fmap=fmap)
+        self.reader = FileReader(fd=fd, fh=fh, read_size=self.reader_block_size, sparse=self.sparse, fmap=fmap, st=st)
 
         # Read the header block first, if there is one; after it, read block_size sized blocks.
         wanted = self.header_size if self.header_size > 0 else self.block_size
diff --git a/src/borg/chunkers/goldilocks_aes.pyi b/src/borg/chunkers/goldilocks_aes.pyi
index aa6c29aaf0..0b9a35357d 100644
--- a/src/borg/chunkers/goldilocks_aes.pyi
+++ b/src/borg/chunkers/goldilocks_aes.pyi
@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import BinaryIO
 
@@ -22,4 +23,6 @@ class ChunkerGoldilocksAES:
         normal_size: int = 0,
         sparse: bool = False,
     ) -> None: ...
-    def chunkify(self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None) -> Iterator: ...
+    def chunkify(
+        self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None, st: os.stat_result = None
+    ) -> Iterator: ...
diff --git a/src/borg/chunkers/rabin_aes.pyi b/src/borg/chunkers/rabin_aes.pyi
index ee52b1a7cb..928f240955 100644
--- a/src/borg/chunkers/rabin_aes.pyi
+++ b/src/borg/chunkers/rabin_aes.pyi
@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import BinaryIO
 
@@ -19,4 +20,6 @@ class ChunkerRabinAES:
         normal_size: int = 0,
         sparse: bool = False,
     ) -> None: ...
-    def chunkify(self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None) -> Iterator: ...
+    def chunkify(
+        self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None, st: os.stat_result = None
+    ) -> Iterator: ...
diff --git a/src/borg/chunkers/reader.pyi b/src/borg/chunkers/reader.pyi
index 7870952f61..83337abb7c 100644
--- a/src/borg/chunkers/reader.pyi
+++ b/src/borg/chunkers/reader.pyi
@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import Any, BinaryIO, NamedTuple
 
@@ -36,6 +37,7 @@ class FileReader:
         read_size: int = 0,
         sparse: bool = False,
         fmap: list[fmap_entry] = None,
+        st: os.stat_result = None,
     ) -> None: ...
     def _fill_buffer(self) -> bool: ...
     def _readinto_direct(self, tv: memoryview, size: int) -> int: ...
diff --git a/src/borg/chunkers/reader.pyx b/src/borg/chunkers/reader.pyx
index f6866de03f..dccfbc9129 100644
--- a/src/borg/chunkers/reader.pyx
+++ b/src/borg/chunkers/reader.pyx
@@ -4,6 +4,7 @@
 
 import os
 import errno
+import stat
 import time
 from collections import namedtuple
 
@@ -227,7 +228,7 @@ class FileReader:
     Most complexity in here comes from the desired size when a user calls FileReader.read does
     not need to match the Chunk sizes we got from the FileFMAPReader.
     """
-    def __init__(self, *, fd=None, fh=-1, read_size=0, sparse=False, fmap=None):
+    def __init__(self, *, fd=None, fh=-1, read_size=0, sparse=False, fmap=None, st=None):
         assert read_size > 0
         self.reader = FileFMAPReader(fd=fd, fh=fh, read_size=read_size, sparse=sparse, fmap=fmap)
         self.buffer = []  # list of Chunk objects
@@ -241,7 +242,22 @@ class FileReader:
         # consider - the file is read start to end. readinto() then reads directly
         # from the file into the caller's buffer (see there), instead of going
         # through the block reader.
-        self.direct = not sparse and fmap is None
+        # Only known regular files take the direct path: reading FIFOs / devices via
+        # --read-special must keep using the proven block reader path - e.g. NetBSD 10
+        # was seen blocking forever in the direct path's big readv() on a FIFO, where
+        # the block reader's 1 MiB os.read() calls work fine.
+        self.direct = False
+        if not sparse and fmap is None:
+            if st is None:
+                # caller did not have a stat result at hand - determine the file type here.
+                try:
+                    if fh >= 0:
+                        st = os.fstat(fh)
+                    elif fd is not None:
+                        st = os.fstat(fd.fileno())
+                except (AttributeError, OSError):
+                    pass  # no OS-level fd (e.g. BytesIO, wrapper objects) - stay on the block reader path
+            self.direct = st is not None and stat.S_ISREG(st.st_mode)
         self.direct_offset = 0  # bytes read so far via the direct path (for fadvise)
 
     def _fill_buffer(self):
@@ -399,9 +415,9 @@ class FileReader:
         Read up to 'size' bytes from the file directly into 'target' (a writable
         buffer, e.g. a memoryview over the caller's scan buffer).
 
-        Fast path (no sparse processing, no fmap given): the file data is read
-        by the OS directly into 'target' - zero copies in user space and one
-        syscall per request instead of one per block.
+        Fast path (known regular file, no sparse processing, no fmap given):
+        the file data is read by the OS directly into 'target' - zero copies
+        in user space and one syscall per request instead of one per block.
 
         Otherwise, unlike read(), this does not allocate or combine intermediate
         byte objects: each byte is copied exactly once, from the buffered file
diff --git a/src/borg/chunkers/toeplitz_aes.pyi b/src/borg/chunkers/toeplitz_aes.pyi
index 82096514ed..4016f34ea9 100644
--- a/src/borg/chunkers/toeplitz_aes.pyi
+++ b/src/borg/chunkers/toeplitz_aes.pyi
@@ -1,3 +1,4 @@
+import os
 from collections.abc import Iterator
 from typing import BinaryIO
 
@@ -21,4 +22,6 @@ class ChunkerToeplitzAES:
         normal_size: int = 0,
         sparse: bool = False,
     ) -> None: ...
-    def chunkify(self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None) -> Iterator: ...
+    def chunkify(
+        self, fd: BinaryIO = None, fh: int = -1, fmap: list[fmap_entry] = None, st: os.stat_result = None
+    ) -> Iterator: ...
diff --git a/src/borg/testsuite/chunkers/reader_test.py b/src/borg/testsuite/chunkers/reader_test.py
index dfebc24b0e..b4bf3a9981 100644
--- a/src/borg/testsuite/chunkers/reader_test.py
+++ b/src/borg/testsuite/chunkers/reader_test.py
@@ -344,3 +344,69 @@ def test_filefmapreader_build_fmap():
     assert fmap[0][0] == 0  # start
     assert fmap[0][1] == 2**62  # size
     assert fmap[0][2] is True  # is_data
+
+
+def test_filereader_readinto_direct(tmpdir):
+    """readinto uses the direct read path for a known regular file (st given)."""
+    fn = str(tmpdir / "regular_file")
+    file_content = b"data1234" * 100
+    with open(fn, "wb") as fd:
+        fd.write(file_content)
+
+    fh = os.open(fn, os.O_RDONLY)
+    try:
+        st = os.fstat(fh)
+        reader = FileReader(fd=None, fh=fh, read_size=1024, sparse=False, fmap=None, st=st)
+        assert reader.direct  # regular file, no sparse, no fmap -> direct read path
+        target = bytearray(len(file_content) + 42)
+        got = reader.readinto(target, len(file_content) + 42)
+        assert got == len(file_content)
+        assert target[:got] == file_content
+        assert reader.readinto(target, 42) == 0  # EOF
+    finally:
+        os.close(fh)
+
+
+def test_filereader_direct_without_st(tmpdir):
+    """Without a given stat result, FileReader stats the file itself: regular file -> direct."""
+    fn = str(tmpdir / "regular_file")
+    file_content = b"data1234"
+    with open(fn, "wb") as fd:
+        fd.write(file_content)
+
+    # OS-level file handle, no st given
+    fh = os.open(fn, os.O_RDONLY)
+    try:
+        reader = FileReader(fd=None, fh=fh, read_size=1024, sparse=False, fmap=None)
+        assert reader.direct  # fstat(fh) says regular file
+    finally:
+        os.close(fh)
+
+    # Python file object, no st given
+    with open(fn, "rb") as fd:
+        reader = FileReader(fd=fd, fh=-1, read_size=1024, sparse=False, fmap=None)
+        assert reader.direct  # fstat(fd.fileno()) says regular file
+        target = bytearray(8)
+        assert reader.readinto(target, 8) == 8
+        assert bytes(target) == file_content
+
+
+def test_filereader_no_direct_without_os_fd():
+    """A file-like object without an OS-level fd must use the buffered block reader path."""
+    file_content = b"data1234"
+    reader = FileReader(fd=BytesIO(file_content), fh=-1, read_size=1024, sparse=False, fmap=None)
+    assert not reader.direct  # no OS-level fd, unknown file type -> buffered path
+    target = bytearray(8)
+    assert reader.readinto(target, 8) == 8
+    assert bytes(target) == file_content
+
+
+def test_filereader_no_direct_for_fifo(tmpdir):
+    """A FIFO's stat result must not enable the direct read path (hung on NetBSD)."""
+    if not hasattr(os, "mkfifo"):
+        pytest.skip("no FIFOs on this platform")
+    fifo_fn = str(tmpdir / "fifo")
+    os.mkfifo(fifo_fn)
+    st = os.stat(fifo_fn)
+    reader = FileReader(fd=BytesIO(b""), fh=-1, read_size=1024, sparse=False, fmap=None, st=st)
+    assert not reader.direct

From 992fc79bc0041e0c91dbb50ad77f5d4c100d210d Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Mon, 10 Aug 2026 01:54:48 +0530
Subject: [PATCH 073/144] check: reflow SIGINT epilog paragraph so --help has
 no short mid-line

---
 src/borg/archiver/check_cmd.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 1c1fa2e80a..fa31c6a98b 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -194,9 +194,8 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         ``--find-lost-archives`` stop after the current chunk; a ``--repair`` archive check
         stops between whole archives. Results recorded before the interrupt are kept, so a later
         check does not re-verify those packs until they are due again. With ``--repair``, an
-        interrupted archive check may leave some archives
-        already repaired and others not yet processed, so run ``borg check --repair`` again to
-        finish.
+        interrupted archive check may leave some archives already repaired and others not yet
+        processed, so run ``borg check --repair`` again to finish.
 
         During a ``--repair`` run, the archive check first rebuilds the chunk index from the
         packs, and, if the key must be recovered, scans chunks for it. These phases do not yet

From da6f388150329460b74358166818b4ca5b8a1c54 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Mon, 10 Aug 2026 02:14:35 +0530
Subject: [PATCH 074/144] check: report invalid pack names instead of crashing

An object in packs/ whose name is not valid hex made check() abort via
hex_to_bin; report it as an error and continue checking the other packs.
---
 src/borg/repository.py                | 14 ++++++++++++++
 src/borg/testsuite/repository_test.py | 15 +++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index 8665bba9c3..71ed4df169 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1,5 +1,6 @@
 import io
 import os
+import re
 import sys
 import threading
 import time
@@ -34,6 +35,9 @@
 
 logger = create_logger(__name__)
 
+# an object name is its sha256 as 64 lowercase hex digits.
+_valid_object_name = re.compile(r"[0-9a-f]{64}").fullmatch
+
 
 def repo_lister(repository, *, limit=None):
     marker = None
@@ -1046,6 +1050,16 @@ def store_list(namespace):
         if index_errors == 0:
             # packs are the bulk of the work and the part --max-duration spreads over several checks.
             pack_infos = store_list("packs")
+            # drop objects whose name is not a valid pack name and count them as errors; the code
+            # below decodes each name via hex_to_bin, which only accepts valid names.
+            valid_pack_infos = []
+            for info in pack_infos:
+                if _valid_object_name(info.name):
+                    valid_pack_infos.append(info)
+                else:
+                    logger.error(f"Store object packs/{info.name} has an invalid name.")
+                    pack_errors += 1
+            pack_infos = valid_pack_infos
             if partial:
                 # a partial check stops after max_duration; verify the least-recently-checked packs
                 # first so repeated runs cover every pack. sort by recorded check time, unrecorded
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index bd1f392ed7..34d39646fb 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1056,6 +1056,21 @@ def test_check_detects_index_corruption(tmp_path):
         assert repository.check(repair=False) is False  # mismatch between content hash and name detected
 
 
+def test_check_reports_invalid_pack_name(tmp_path, caplog):
+    # an object in packs/ whose name is not 64 hex digits is reported as an error, and the other
+    # packs are still checked.
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        intact_id, _ = _store_intact_pack(repository)
+        repository.store_store("packs/not-a-hex-name", b"stray junk")
+
+        with caplog.at_level(logging.ERROR, logger="borg.repository"):
+            assert repository.check(repair=False) is False
+
+        assert "packs/not-a-hex-name has an invalid name" in caplog.text
+        after = PackTracker.load(repository.store)
+        assert after.table[intact_id].result == 1  # the valid pack was checked
+
+
 def test_check_warns_on_invalid_chunk_index(tmp_path, caplog):
     # check warns about an invalid chunk index but does not fail, since the index is not part of
     # the repository's object integrity.

From 4211d49aa3e4a90a1f5cc8c56a3d93b6296cc4c5 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Mon, 10 Aug 2026 07:35:41 +0530
Subject: [PATCH 075/144] check --repair: flush pack writer in
 ArchiveChecker.finish() before dropping the index

During --repair, add_reference() re-adds chunks via repository.put(), which only
buffers them in the pack writer. finish() then deleted the chunk index without
flushing, so a buffered chunk survived to repository.close(), which asserts an
empty buffer. Flush in finish() before deleting the index.
---
 src/borg/archive.py                           |  3 ++
 src/borg/testsuite/archiver/check_cmd_test.py | 28 +++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 727b6e5cac..1c41536354 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -2392,6 +2392,9 @@ def valid_item(obj):
 
     def finish(self):
         if self.repair:
+            # store packs still buffered from chunks re-added during repair, so their index entries
+            # are set before the index is dropped below and no chunk is left buffered at close().
+            self.repository.flush()
             # we may have deleted chunks. delete_chunkindex_from_repo() removes the on-disk index and
             # drops the stale in-memory index, so the next repository access rebuilds it from the repo.
             logger.info("Deleting chunk indexes in repository - next repository access will cause a rebuild.")
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index 03d7feba6e..d7af0e5a1e 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -556,6 +556,34 @@ def test_extra_chunks(archivers, request):
     cmd(archiver, "check", "-v", exit_code=0)  # check does not deal with orphans anymore
 
 
+def test_repair_finish_flushes_pack_writer(archivers, request):
+    """finish() stores chunks re-added during --repair before it drops the index (#10055).
+
+    close() asserts an empty pack writer buffer, so a chunk left buffered by finish() would
+    trip it.
+    """
+    archiver = request.getfixturevalue(archivers)
+    if archiver.get_kind() != "local":
+        pytest.skip("inspects in-process repository internals")
+    check_cmd_setup(archiver)
+    cmd(archiver, "check", exit_code=0)
+
+    with Repository(archiver.repository_location, exclusive=True) as repository:
+        checker = ArchiveChecker()
+        checker.repair = True
+        checker.repository = repository
+        checker.key = checker.make_key(repository)
+        checker.manifest = Manifest.load(repository, (Manifest.Operation.CHECK,), key=checker.key)
+
+        # a chunk re-added during repair, buffered in the pack writer:
+        key = b"01234567890123456789012345678901"
+        repository.put(key, fchunk(b"repaired", chunk_id=key))
+        assert repository._pack_writer._pieces
+
+        checker.finish()
+        assert not repository._pack_writer._pieces  # finish() stored it
+
+
 @pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none"]])
 def test_verify_data(archivers, request, init_args):
     archiver = request.getfixturevalue(archivers)

From a8a2084e36ed1e7f811ca78f987008a4582bbc57 Mon Sep 17 00:00:00 2001
From: ThomasWaldmann <356103+ThomasWaldmann@users.noreply.github.com>
Date: Mon, 10 Aug 2026 04:50:55 +0000
Subject: [PATCH 076/144] FAME.md: update contributor statistics

---
 FAME.md  | 42 +++++++++++++++++++++---------------------
 FAME.svg | 16 ++++++++--------
 2 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/FAME.md b/FAME.md
index 54fc3ffa93..4a45d90cf0 100644
--- a/FAME.md
+++ b/FAME.md
@@ -1,11 +1,11 @@
 # Contributors
 
-372 people have contributed to Borg, with 10,607 commits in total.
+372 people have contributed to Borg, with 10,733 commits in total.
 Thanks to everyone who helped!
 
 ![Contributors by commit count](FAME.svg)
 
-Generated on 2026-08-03 by `scripts/fame.py`, which computes the statistics with
+Generated on 2026-08-10 by `scripts/fame.py`, which computes the statistics with
 [git-fame](https://github.com/casperdcl/git-fame), from the `master` branch
 only - commits that exist solely on other branches or in unmerged pull requests
 are not counted.
@@ -21,30 +21,30 @@ show up there, so the number understates early contributions.  Generated files
 
 | Contributor | Commits | Lines | Files |
 |:---|---:|---:|---:|
-| Thomas Waldmann | 7,044 | 76,773 | 461 |
-| Marian Beermann | 1,140 | 8,400 | 157 |
-| Jonas Borgström | 560 | 1,028 | 38 |
+| Thomas Waldmann | 7,148 | 82,508 | 490 |
+| Marian Beermann | 1,140 | 8,158 | 156 |
+| Jonas Borgström | 560 | 986 | 38 |
 | Antoine Beaupré | 285 | 441 | 33 |
-| Mrityunjay Raj | 198 | 7,917 | 102 |
+| Mrityunjay Raj | 217 | 8,493 | 102 |
 | Andrey Bienkowski | 72 | 210 | 16 |
 | Ted Lawson | 71 | 4,606 | 39 |
-| Thalian | 63 | 594 | 32 |
+| Thalian | 63 | 584 | 31 |
 | Martin Hostettler | 46 | 2,559 | 9 |
-| Milkey Mouse | 42 | 248 | 14 |
+| Milkey Mouse | 42 | 245 | 13 |
 | Dan Christensen | 40 | 5 | 2 |
-| dependabot[bot] | 37 | 46 | 7 |
+| dependabot[bot] | 39 | 46 | 7 |
 | Rayyan Ansari | 34 | 91 | 12 |
 | Abdel-Rahman | 30 | 3 | 3 |
 | Manuel Riel | 28 | 94 | 6 |
 | Nehalenniæ Oudin | 26 | 48 | 11 |
 | anarcat | 24 | 34 | 4 |
-| Hugo Wallenburg | 23 | 1,189 | 13 |
+| Hugo Wallenburg | 23 | 1,183 | 12 |
 | Michael Hanselmann | 22 | 71 | 3 |
-| SanskritFritz | 17 | 258 | 1 |
 | Björn Ketelaars | 17 | 33 | 6 |
+| SanskritFritz | 17 | 0 | 0 |
 | Lee Bousfield | 14 | 160 | 9 |
-| Alan Jenkins | 14 | 38 | 2 |
-| Daniel Rudolf | 13 | 230 | 14 |
+| Alan Jenkins | 14 | 35 | 2 |
+| Daniel Rudolf | 13 | 220 | 14 |
 | Robin Schneider | 13 | 19 | 8 |
 | Elmar Hoffmann | 12 | 90 | 6 |
 | Jürg Rast | 12 | 73 | 11 |
@@ -71,7 +71,7 @@ show up there, so the number understates early contributions.  Generated files
 | Felix Schwarz | 6 | 4 | 2 |
 | Alexander-N | 6 | 0 | 0 |
 | Paul D | 5 | 185 | 46 |
-| Simon Frei | 5 | 174 | 3 |
+| Simon Frei | 5 | 132 | 3 |
 | Dominik Stadler | 5 | 77 | 1 |
 | elandorr | 5 | 40 | 3 |
 | remyabel | 5 | 25 | 3 |
@@ -86,7 +86,6 @@ show up there, so the number understates early contributions.  Generated files
 | Franco Ayala | 4 | 39 | 6 |
 | Suryansh Pal | 4 | 36 | 9 |
 | 8bit | 4 | 30 | 5 |
-| oxiedi | 4 | 26 | 1 |
 | Steve Groesz | 4 | 22 | 1 |
 | Ryan Polley | 4 | 12 | 4 |
 | Artem Sheremet | 4 | 10 | 1 |
@@ -94,6 +93,7 @@ show up there, so the number understates early contributions.  Generated files
 | Josh Holland | 4 | 3 | 2 |
 | Narendra Vardi | 4 | 3 | 1 |
 | Alexander Pyhalov | 4 | 0 | 0 |
+| oxiedi | 4 | 0 | 0 |
 | Stephan Herbers | 3 | 49 | 2 |
 | Charmi Kadi | 3 | 46 | 2 |
 | Martin Richtarsky | 3 | 34 | 4 |
@@ -103,7 +103,7 @@ show up there, so the number understates early contributions.  Generated files
 | Michael Gajda | 3 | 23 | 2 |
 | eoli3n | 3 | 22 | 1 |
 | Will | 3 | 21 | 4 |
-| Andrea Gelmini | 3 | 18 | 13 |
+| Andrea Gelmini | 3 | 17 | 12 |
 | Oleg Drokin | 3 | 16 | 1 |
 | kmq | 3 | 15 | 1 |
 | jungle-boogie | 3 | 13 | 1 |
@@ -125,12 +125,13 @@ show up there, so the number understates early contributions.  Generated files
 | jhemmje | 3 | 0 | 0 |
 | Thomas Portmann | 2 | 206 | 5 |
 | Ken Kundert | 2 | 149 | 4 |
+| ThomasWaldmann | 2 | 145 | 1 |
 | trxvorr | 2 | 103 | 7 |
 | Syed Ali Ghazi Ejaz | 2 | 90 | 4 |
 | David Rambo | 2 | 87 | 3 |
-| Soumik Dutta | 2 | 85 | 3 |
-| Eric Wolf | 2 | 51 | 4 |
+| Soumik Dutta | 2 | 80 | 3 |
 | Divyansh Agrawal | 2 | 43 | 4 |
+| Eric Wolf | 2 | 38 | 3 |
 | Ioannis Cherouvim | 2 | 32 | 2 |
 | zDEFz | 2 | 27 | 1 |
 | Lapinot | 2 | 24 | 4 |
@@ -200,7 +201,6 @@ show up there, so the number understates early contributions.  Generated files
 | Mike Mason | 1 | 38 | 3 |
 | borkd | 1 | 37 | 1 |
 | James Vasile | 1 | 36 | 1 |
-| lexa-a | 1 | 31 | 1 |
 | Uriel | 1 | 30 | 1 |
 | vancheese | 1 | 28 | 8 |
 | Nic Donaldson | 1 | 26 | 1 |
@@ -208,7 +208,6 @@ show up there, so the number understates early contributions.  Generated files
 | Mike | 1 | 23 | 1 |
 | Sitaram Chamarty | 1 | 23 | 1 |
 | Jonathan Zacsh | 1 | 21 | 1 |
-| edvatar | 1 | 16 | 2 |
 | Félix Sipma | 1 | 13 | 1 |
 | remyabel2 | 1 | 13 | 1 |
 | Antonio Larrosa | 1 | 12 | 1 |
@@ -222,7 +221,7 @@ show up there, so the number understates early contributions.  Generated files
 | Jubjub | 1 | 6 | 1 |
 | Mher Kazandjian | 1 | 6 | 1 |
 | TawfeeqShaik | 1 | 6 | 1 |
-| ThomasWaldmann | 1 | 6 | 1 |
+| edvatar | 1 | 6 | 2 |
 | Florent Hemmi | 1 | 5 | 1 |
 | Leo Antunes | 1 | 5 | 1 |
 | dataprolet | 1 | 5 | 1 |
@@ -381,6 +380,7 @@ show up there, so the number understates early contributions.  Generated files
 | jeroen tiebout | 1 | 0 | 0 |
 | kannes | 1 | 0 | 0 |
 | klemens | 1 | 0 | 0 |
+| lexa-a | 1 | 0 | 0 |
 | lumbric | 1 | 0 | 0 |
 | mirobertod | 1 | 0 | 0 |
 | nain-F49FF806 | 1 | 0 | 0 |
diff --git a/FAME.svg b/FAME.svg
index 7a695be0c9..6822437656 100644
--- a/FAME.svg
+++ b/FAME.svg
@@ -19,19 +19,19 @@
 <text class="subtitle" x="14" y="44">top 20 of 371 by commits on the master branch</text>
 <text class="name" x="14" y="73">Thomas Waldmann</text>
 <rect class="bar" x="240" y="63" width="436" height="12" rx="2"/>
-<text class="count" x="746" y="73">7,044</text>
+<text class="count" x="746" y="73">7,148</text>
 <text class="name" x="14" y="95">Marian Beermann</text>
-<rect class="bar" x="240" y="85" width="71" height="12" rx="2"/>
+<rect class="bar" x="240" y="85" width="70" height="12" rx="2"/>
 <text class="count" x="746" y="95">1,140</text>
 <text class="name" x="14" y="117">Jonas Borgström</text>
-<rect class="bar" x="240" y="107" width="35" height="12" rx="2"/>
+<rect class="bar" x="240" y="107" width="34" height="12" rx="2"/>
 <text class="count" x="746" y="117">560</text>
 <text class="name" x="14" y="139">Antoine Beaupré</text>
-<rect class="bar" x="240" y="129" width="18" height="12" rx="2"/>
+<rect class="bar" x="240" y="129" width="17" height="12" rx="2"/>
 <text class="count" x="746" y="139">285</text>
 <text class="name" x="14" y="161">Mrityunjay Raj</text>
-<rect class="bar" x="240" y="151" width="12" height="12" rx="2"/>
-<text class="count" x="746" y="161">198</text>
+<rect class="bar" x="240" y="151" width="13" height="12" rx="2"/>
+<text class="count" x="746" y="161">217</text>
 <text class="name" x="14" y="183">Andrey Bienkowski</text>
 <rect class="bar" x="240" y="173" width="4" height="12" rx="2"/>
 <text class="count" x="746" y="183">72</text>
@@ -71,10 +71,10 @@
 <text class="name" x="14" y="447">Michael Hanselmann</text>
 <rect class="bar" x="240" y="437" width="2" height="12" rx="2"/>
 <text class="count" x="746" y="447">22</text>
-<text class="name" x="14" y="469">SanskritFritz</text>
+<text class="name" x="14" y="469">Björn Ketelaars</text>
 <rect class="bar" x="240" y="459" width="2" height="12" rx="2"/>
 <text class="count" x="746" y="469">17</text>
-<text class="name" x="14" y="491">Björn Ketelaars</text>
+<text class="name" x="14" y="491">SanskritFritz</text>
 <rect class="bar" x="240" y="481" width="2" height="12" rx="2"/>
 <text class="count" x="746" y="491">17</text>
 <text class="footer" x="14" y="514">generated by scripts/fame.py using git-fame</text>

From 67e05732b3e57f3316e9ea1b674fd6798ec2e83e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 11 Aug 2026 05:43:17 +0000
Subject: [PATCH 077/144] build(deps-dev): bump the pip-dependencies group

Bumps the pip-dependencies group in /requirements.d with 3 updates: [virtualenv](https://github.com/pypa/virtualenv), [coverage](https://github.com/coveragepy/coveragepy) and [pre-commit](https://github.com/pre-commit/pre-commit).


Updates `virtualenv` from 21.5.2 to 21.6.1
- [Release notes](https://github.com/pypa/virtualenv/releases)
- [Changelog](https://github.com/pypa/virtualenv/blob/main/docs/changelog.rst)
- [Commits](https://github.com/pypa/virtualenv/compare/21.5.2...21.6.1)

Updates `coverage` from 7.15.0 to 7.15.4
- [Release notes](https://github.com/coveragepy/coveragepy/releases)
- [Changelog](https://github.com/coveragepy/coveragepy/blob/main/CHANGES.rst)
- [Commits](https://github.com/coveragepy/coveragepy/compare/7.15.0...7.15.4)

Updates `pre-commit` from 4.6.1 to 4.6.2
- [Release notes](https://github.com/pre-commit/pre-commit/releases)
- [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md)
- [Commits](https://github.com/pre-commit/pre-commit/compare/v4.6.1...v4.6.2)

---
updated-dependencies:
- dependency-name: virtualenv
  dependency-version: 21.6.1
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: pip-dependencies
- dependency-name: coverage
  dependency-version: 7.15.4
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: pip-dependencies
- dependency-name: pre-commit
  dependency-version: 4.6.2
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: pip-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.d/development.lock.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.d/development.lock.txt b/requirements.d/development.lock.txt
index 0e9a661c5f..148b1d461d 100644
--- a/requirements.d/development.lock.txt
+++ b/requirements.d/development.lock.txt
@@ -1,12 +1,12 @@
 pip==26.1.2
-virtualenv==21.5.2
+virtualenv==21.6.1
 build==1.5.1
 tox==4.56.4
 pytest==9.1.1
 pytest-xdist==3.8.0
-coverage[toml]==7.15.0
+coverage[toml]==7.15.4
 pytest-cov==7.1.0
 pytest-benchmark==5.2.3
 pytest-timeout==2.4.0
-pre-commit==4.6.1
+pre-commit==4.6.2
 types-PyYAML==6.0.12.20260724

From 3f9c362e4f9733b783aba0a781a46fd43011f083 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 11 Aug 2026 05:45:18 +0000
Subject: [PATCH 078/144] build(deps): bump github/codeql-action in the actions
 group

Bumps the actions group with 1 update: [github/codeql-action](https://github.com/github/codeql-action).


Updates `github/codeql-action` from 4.37.4 to 4.37.6
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/v4.37.4...v4.37.6)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-version: 4.37.6
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: actions
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/codeql-analysis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 869a598e0c..b4be4306cb 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -69,7 +69,7 @@ jobs:
         sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v4.37.4
+      uses: github/codeql-action/init@v4.37.6
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -83,4 +83,4 @@ jobs:
         pip3 install -r requirements.d/development.txt
         pip3 install -ve .
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v4.37.4
+      uses: github/codeql-action/analyze@v4.37.6

From c058be33ae2bb9e745a6a2a2bd0458b5b48753c4 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Mon, 10 Aug 2026 20:09:57 +0200
Subject: [PATCH 079/144] CI: upgrade cross-platform-actions to 1.4.0, NetBSD
 to 11.0

cross-platform-actions 1.4.0 adds NetBSD 11.0 VM images and bounds
waiting for VM boot by a wall-clock timeout, so a VM that never becomes
ssh-reachable fails early instead of retrying for hours.

Also point the pkgin repository at the NetBSD 11.0 pkgsrc package set
(py311 pip/virtualenv/tox and all other used packages are available
there).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f473fef4e6..cb65b9e28a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -411,7 +411,7 @@ jobs:
             artifact_prefix: borg-freebsd-15-x86_64-gh
 
           - os: netbsd
-            version: '10.1'
+            version: '11.0'
             display_name: NetBSD
             do_binaries: false
 
@@ -434,10 +434,10 @@ jobs:
 
       - name: Start VM for ${{ matrix.display_name }}
         id: cross_os
-        # a normal boot takes < 1 minute; sometimes the VM never becomes
-        # ssh-reachable and the action retries forever - fail early then.
+        # a normal boot takes < 1 minute; since 1.4.0 the action bounds
+        # waiting for boot itself, this is just an additional safety net.
         timeout-minutes: 15
-        uses: cross-platform-actions/action@v1.3.0
+        uses: cross-platform-actions/action@v1.4.0
         with:
           operating_system: ${{ matrix.os }}
           version: ${{ matrix.version }}
@@ -512,7 +512,7 @@ jobs:
             netbsd)
               arch="$(uname -m)"
               sudo -E mkdir -p /usr/pkg/etc/pkgin
-              echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
+              echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/11.0/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
               sudo -E pkgin update
               sudo -E pkgin -y upgrade
               sudo -E pkgin -y install lz4 git

From 5de8211933fe0591a67687a2734f0d3fa4dfc5ea Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 10:14:27 +0200
Subject: [PATCH 080/144] tests: time-bound LRUCache.test_threaded_access

With the tiny switch interval (1e-6s) and line-by-line coverage
tracing, the fixed 8 threads x 2000 rounds x 16 keys workload takes
wildly different time across platforms - on the NetBSD 11 CI VM it
blew the 300s pytest-timeout (the threads were alive, just slow).

Loop until a 2s deadline instead: fast machines still hammer the
cache with many rounds, slow ones just do fewer.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/testsuite/helpers/lrucache_test.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/borg/testsuite/helpers/lrucache_test.py b/src/borg/testsuite/helpers/lrucache_test.py
index 1462098bfa..4a1825e395 100644
--- a/src/borg/testsuite/helpers/lrucache_test.py
+++ b/src/borg/testsuite/helpers/lrucache_test.py
@@ -1,5 +1,6 @@
 import sys
 import threading
+import time
 from tempfile import TemporaryFile
 
 import pytest
@@ -105,14 +106,21 @@ def test_threaded_access(self):
         errors: list[Exception] = []
         start = threading.Barrier(8)
 
+        # time-bounded instead of a fixed round count: with the tiny switch interval and
+        # coverage tracing, a fixed count takes wildly different time across platforms
+        # (it ran into the test timeout on a slow NetBSD CI VM).
+        deadline = time.monotonic() + 2.0
+
         def worker(n):
             start.wait()
             try:
-                for _round in range(2000):
+                while True:
                     for key in keys:
                         c[key] = n
                         c.get(key)
                         c.pop(key, None)
+                    if time.monotonic() >= deadline:
+                        break
             except Exception as e:
                 errors.append(e)
 

From 9e552cc89e5952ed1f47f1c65d26407d360774fc Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Mon, 10 Aug 2026 20:44:03 +0200
Subject: [PATCH 081/144] CI: NetBSD: use Python 3.13

The rust package already depends on python313, so using py313
avoids installing a second Python.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cb65b9e28a..0787326f5d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -518,11 +518,12 @@ jobs:
               sudo -E pkgin -y install lz4 git
               sudo -E pkgin -y install rust
               sudo -E pkgin -y install pkg-config
-              sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox
-              sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3
-              sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3
-              sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3
-              sudo -E ln -sf /usr/pkg/bin/tox-3.11 /usr/pkg/bin/tox3
+              # rust already depends on python313, so use that
+              sudo -E pkgin -y install py313-pip py313-virtualenv py313-tox
+              sudo -E ln -sf /usr/pkg/bin/python3.13 /usr/pkg/bin/python3
+              sudo -E ln -sf /usr/pkg/bin/pip3.13 /usr/pkg/bin/pip3
+              sudo -E ln -sf /usr/pkg/bin/virtualenv-3.13 /usr/pkg/bin/virtualenv3
+              sudo -E ln -sf /usr/pkg/bin/tox-3.13 /usr/pkg/bin/tox3
 
               # Ensure base system admin tools are on PATH for the non-root shell
               export PATH="/sbin:/usr/sbin:$PATH"
@@ -545,7 +546,7 @@ jobs:
               touch ${TMPDIR}/testfile
               lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***"
 
-              tox3 -e py311-none
+              tox3 -e py313-none
               ;;
 
             openbsd)

From 5509d2df0a5cf87645126d29ad751ede2d751499 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Mon, 10 Aug 2026 20:44:03 +0200
Subject: [PATCH 082/144] NetBSD 11: / has EAs enabled, use a TMPDIR for tests
 there

/tmp is a tmpfs without EAs, we can't use that for our tests.
---
 .github/workflows/ci.yml | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0787326f5d..16b66dadda 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -528,18 +528,8 @@ jobs:
               # Ensure base system admin tools are on PATH for the non-root shell
               export PATH="/sbin:/usr/sbin:$PATH"
 
-              echo "--- Preparing an extattr-enabled filesystem ---"
-              # On many NetBSD setups /tmp is tmpfs without extended attributes.
-              # Create a FFS image with extended attributes enabled and use it for TMPDIR.
-              VNDDEV="vnd0"
-              IMGFILE="/tmp/fs.img"
-              sudo -E dd if=/dev/zero of=${IMGFILE} bs=1m count=1024
-              sudo -E vndconfig -c "${VNDDEV}" "${IMGFILE}"
-              sudo -E newfs -O 2ea /dev/r${VNDDEV}a
-              MNT="/mnt/eafs"
-              sudo -E mkdir -p ${MNT}
-              sudo -E mount -t ffs -o extattr /dev/${VNDDEV}a $MNT
-              export TMPDIR="${MNT}/tmp"
+              # On the netbsd 11 VM, / is a fs with extended attributes.
+              export TMPDIR="/tmp_eafs"
               sudo -E mkdir -p ${TMPDIR}
               sudo -E chmod 1777 ${TMPDIR}
 

From c9f03ef7715399750be169ddac27b7a002246dc5 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 11:40:13 +0200
Subject: [PATCH 083/144] tests: compute the TMPDIR-based runtime dir fallback

platformdirs falls back to {tempfile.gettempdir()}/runtime-{uid} when
the platform default runtime dir is not writable, so compute that
instead of hardcoding each CI's TMPDIR in the allowlist (the NetBSD
job's TMPDIR just changed to /tmp_eafs and broke the test again).

Also add OpenBSD's platform default /tmp/run/user/{uid}.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/testsuite/helpers/fs_test.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/borg/testsuite/helpers/fs_test.py b/src/borg/testsuite/helpers/fs_test.py
index 6737ad9157..2eec3de238 100644
--- a/src/borg/testsuite/helpers/fs_test.py
+++ b/src/borg/testsuite/helpers/fs_test.py
@@ -1,6 +1,7 @@
 import errno
 import os
 import sys
+import tempfile
 from contextlib import contextmanager
 from pathlib import Path
 
@@ -177,11 +178,11 @@ def test_get_runtime_dir(monkeypatch):
         monkeypatch.delenv("BORG_RUNTIME_DIR", raising=False)
         uid = str(os.getuid())
         assert get_runtime_dir(create=False) in [
-            os.path.join("/run/user", uid, "borg"),
-            os.path.join("/var/run/user", uid, "borg"),
-            os.path.join(f"/tmp/runtime-{uid}", "borg"),
-            os.path.join(f"/mnt/eafs/tmp/runtime-{uid}", "borg"),  # CI netbsd
-            os.path.join(f"/var/tmp/borg-ci/runtime-{uid}", "borg"),  # CI omnios (TMPDIR)
+            os.path.join("/run/user", uid, "borg"),  # Linux
+            os.path.join("/var/run/user", uid, "borg"),  # FreeBSD/NetBSD
+            os.path.join("/tmp/run/user", uid, "borg"),  # OpenBSD
+            # platformdirs falls back to a TMPDIR-based path if the platform default is not writable
+            os.path.join(tempfile.gettempdir(), f"runtime-{uid}", "borg"),
         ]
         monkeypatch.setenv("XDG_RUNTIME_DIR", "/var/tmp/.cache")
         assert get_runtime_dir(create=False) == os.path.join("/var/tmp/.cache", "borg")

From eae8f6da38514a8142effacced8b8e5972c115c5 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 13:40:20 +0200
Subject: [PATCH 084/144] export-tar/import-tar: zstd (de)compression
 in-process, see #10067

With --tar-filter=auto, .tar.zst/.tar.zstd/.tzst tarballs (.tzst newly
recognized) are now (de)compressed in-process using the zstd module
(Python >= 3.14 stdlib / backports.zstd) instead of piping through an
external zstd process. borg requires the zstd module anyway, libzstd
multithreading (BORG_ZSTD_MT_WORKERS) runs outside the GIL, and no
external tool needs to be installed.

Compression uses level 3 (the zstd tool's default). An explicit
--tar-filter program is still run as an external filter, also for zstd.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/usage/export-tar.rst.inc                |  6 +-
 docs/usage/import-tar.rst.inc                |  6 +-
 src/borg/archiver/tar_cmds.py                | 78 ++++++++++++++++++--
 src/borg/testsuite/archiver/tar_cmds_test.py | 69 +++++++++++++++++
 4 files changed, 151 insertions(+), 8 deletions(-)

diff --git a/docs/usage/export-tar.rst.inc b/docs/usage/export-tar.rst.inc
index ad1f6a635c..7bb8cb19f5 100644
--- a/docs/usage/export-tar.rst.inc
+++ b/docs/usage/export-tar.rst.inc
@@ -95,9 +95,13 @@ before writing it to FILE:
 - .tar.gz or .tgz: gzip
 - .tar.bz2 or .tbz: bzip2
 - .tar.xz or .txz: xz
-- .tar.zstd or .tar.zst: zstd
+- .tar.zstd, .tar.zst or .tzst: zstd (in-process, level 3)
 - .tar.lz4: lz4
 
+For zstd, Borg compresses in-process using libzstd instead of piping through an
+external program (set BORG_ZSTD_MT_WORKERS to use multiple compression threads).
+The other formats are piped through the respective external filter program.
+
 Alternatively, a ``--tar-filter`` program may be explicitly specified. It should
 read the uncompressed tar stream from stdin and write a compressed/filtered
 tar stream to stdout.
diff --git a/docs/usage/import-tar.rst.inc b/docs/usage/import-tar.rst.inc
index 64ad82ad13..12a949d678 100644
--- a/docs/usage/import-tar.rst.inc
+++ b/docs/usage/import-tar.rst.inc
@@ -96,9 +96,13 @@ based on its file extension and pipe the file through an appropriate filter:
 - .tar.gz or .tgz: gzip -d
 - .tar.bz2 or .tbz: bzip2 -d
 - .tar.xz or .txz: xz -d
-- .tar.zstd or .tar.zst: zstd -d
+- .tar.zstd, .tar.zst or .tzst: zstd (in-process)
 - .tar.lz4: lz4 -d
 
+For zstd, Borg decompresses in-process using libzstd instead of piping through
+an external program. The other formats are piped through the respective
+external filter program.
+
 Alternatively, a --tar-filter program may be explicitly specified. It should
 read compressed data from stdin and output an uncompressed tar stream on
 stdout.
diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py
index d085ffdbe9..13c4967426 100644
--- a/src/borg/archiver/tar_cmds.py
+++ b/src/borg/archiver/tar_cmds.py
@@ -1,11 +1,20 @@
 import base64
+import contextlib
 import logging
 import os
 import stat
+import sys
 import tarfile
 
+if sys.version_info >= (3, 14):
+    from compression import zstd
+else:
+    from backports import zstd
+
 from ..archive import Archive, TarfileObjectProcessors, ChunksProcessor
+from ..compress import get_zstd_mt_workers
 from ..constants import *  # NOQA
+from ..helpers import Error
 from ..helpers import HardLinkManager, IncludePatternNeverMatchedWarning
 from ..helpers import ProgressIndicatorPercent
 from ..helpers import dash_open
@@ -146,6 +155,14 @@ def item_to_paxheaders(format, item):
     return ph
 
 
+# Sentinel returned by get_tar_filter for zstd suffixes: (de)compress in-process
+# via the zstd module instead of piping through an external filter program.
+IN_PROCESS_ZSTD = "zstd (in-process)"
+
+# Default zstd compression level, same as the zstd command line tool's default.
+ZSTD_TAR_LEVEL = 3
+
+
 def get_tar_filter(fname, decompress):
     # Note that filter is None if fname is '-'.
     if fname.endswith((".tar.gz", ".tgz")):
@@ -156,14 +173,39 @@ def get_tar_filter(fname, decompress):
         filter = "xz -d" if decompress else "xz"
     elif fname.endswith((".tar.lz4",)):
         filter = "lz4 -d" if decompress else "lz4"
-    elif fname.endswith((".tar.zstd", ".tar.zst")):
-        filter = "zstd -d" if decompress else "zstd"
+    elif fname.endswith((".tar.zstd", ".tar.zst", ".tzst")):
+        filter = IN_PROCESS_ZSTD
     else:
         filter = None
     logger.debug("Automatically determined tar filter: %s", filter)
     return filter
 
 
+@contextlib.contextmanager
+def create_zstd_filter(stream, stream_close, decompress):
+    """In-process zstd (de)compression wrapper around *stream*, same contract as create_filter_process."""
+    if decompress:
+        zstream = zstd.ZstdFile(stream, "rb")
+    else:
+        workers = get_zstd_mt_workers()
+        if workers > 1:
+            params = zstd.CompressionParameter
+            options = {params.compression_level: ZSTD_TAR_LEVEL, params.nb_workers: workers}
+            zstream = zstd.ZstdFile(stream, "wb", options=options)
+        else:
+            zstream = zstd.ZstdFile(stream, "wb", level=ZSTD_TAR_LEVEL)
+    try:
+        yield zstream
+    except zstd.ZstdError as e:
+        raise Error(f"in-process zstd filter failed: {e}") from None
+    finally:
+        # for compression, closing also finishes the zstd frame.
+        # ZstdFile does not close a fileobj it was given, so close stream separately.
+        zstream.close()
+        if stream_close:
+            stream.close()
+
+
 class TarMixIn:
     @with_repository(compatibility=(Manifest.Operation.READ,))
     @with_archive
@@ -188,13 +230,25 @@ def do_export_tar(self, args, repository, manifest, archive):
         # none of these drawbacks. The only issue of using an external filter is
         # that it has to be installed -- hardly a problem, considering that
         # the decompressor must be installed as well to make use of the exported tarball!
+        #
+        # zstd is the exception: borg requires the zstd module anyway (Python >= 3.14
+        # stdlib, backports.zstd otherwise), libzstd multithreading (see the
+        # BORG_ZSTD_MT_WORKERS env var) runs outside the GIL, and there is no
+        # significantly more optimized external tool - so zstd tarballs are
+        # (de)compressed in-process and need no external program.
 
         filter = get_tar_filter(args.tarfile, decompress=False) if args.tar_filter == "auto" else args.tar_filter
 
         tarstream = dash_open(args.tarfile, "wb")
         tarstream_close = args.tarfile != "-"
 
-        with create_filter_process(filter, stream=tarstream, stream_close=tarstream_close, inbound=False) as _stream:
+        if filter is IN_PROCESS_ZSTD:
+            filter_context = create_zstd_filter(tarstream, stream_close=tarstream_close, decompress=False)
+        else:
+            filter_context = create_filter_process(
+                filter, stream=tarstream, stream_close=tarstream_close, inbound=False
+            )
+        with filter_context as _stream:
             self._export_tar(args, archive, _stream)
 
     def _export_tar(self, args, archive, tarstream):
@@ -267,7 +321,11 @@ def do_import_tar(self, args, repository, manifest, cache):
         tarstream = dash_open(args.tarfile, "rb")
         tarstream_close = args.tarfile != "-"
 
-        with create_filter_process(filter, stream=tarstream, stream_close=tarstream_close, inbound=True) as _stream:
+        if filter is IN_PROCESS_ZSTD:
+            filter_context = create_zstd_filter(tarstream, stream_close=tarstream_close, decompress=True)
+        else:
+            filter_context = create_filter_process(filter, stream=tarstream, stream_close=tarstream_close, inbound=True)
+        with filter_context as _stream:
             self._import_tar(args, repository, manifest, manifest.key, cache, _stream)
 
     def _import_tar(self, args, repository, manifest, key, cache, tarstream):
@@ -350,9 +408,13 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser):
         - .tar.gz or .tgz: gzip
         - .tar.bz2 or .tbz: bzip2
         - .tar.xz or .txz: xz
-        - .tar.zstd or .tar.zst: zstd
+        - .tar.zstd, .tar.zst or .tzst: zstd (in-process, level 3)
         - .tar.lz4: lz4
 
+        For zstd, Borg compresses in-process using libzstd instead of piping through an
+        external program (set BORG_ZSTD_MT_WORKERS to use multiple compression threads).
+        The other formats are piped through the respective external filter program.
+
         Alternatively, a ``--tar-filter`` program may be explicitly specified. It should
         read the uncompressed tar stream from stdin and write a compressed/filtered
         tar stream to stdout.
@@ -425,9 +487,13 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser):
         - .tar.gz or .tgz: gzip -d
         - .tar.bz2 or .tbz: bzip2 -d
         - .tar.xz or .txz: xz -d
-        - .tar.zstd or .tar.zst: zstd -d
+        - .tar.zstd, .tar.zst or .tzst: zstd (in-process)
         - .tar.lz4: lz4 -d
 
+        For zstd, Borg decompresses in-process using libzstd instead of piping through
+        an external program. The other formats are piped through the respective
+        external filter program.
+
         Alternatively, a --tar-filter program may be explicitly specified. It should
         read compressed data from stdin and output an uncompressed tar stream on
         stdout.
diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py
index cbebdfc2e5..a40c224866 100644
--- a/src/borg/testsuite/archiver/tar_cmds_test.py
+++ b/src/borg/testsuite/archiver/tar_cmds_test.py
@@ -26,6 +26,9 @@ def have_gnutar():
 
 requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason="GNU tar must be installed for this test.")
 requires_gzip = pytest.mark.skipif(not shutil.which("gzip"), reason="gzip must be installed for this test.")
+requires_zstd = pytest.mark.skipif(not shutil.which("zstd"), reason="zstd must be installed for this test.")
+
+ZSTD_MAGIC = b"\x28\xb5\x2f\xfd"
 
 
 @requires_gnutar
@@ -159,6 +162,72 @@ def test_import_tar_gz(archivers, request, tar_format="GNU"):
     assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True)
 
 
+@pytest.mark.parametrize("suffix", ["tar.zst", "tar.zstd", "tzst"])
+def test_export_import_tar_zst(archivers, request, suffix):
+    # zstd tarballs are (de)compressed in-process, no external zstd binary is needed.
+    archiver = request.getfixturevalue(archivers)
+    create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
+    os.unlink("input/flagfile")
+    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "create", "src", "input")
+    cmd(archiver, "export-tar", "src", f"simple.{suffix}")
+    with open(f"simple.{suffix}", "rb") as fd:
+        assert fd.read(4) == ZSTD_MAGIC
+    cmd(archiver, "import-tar", "dst", f"simple.{suffix}")
+    with changedir(archiver.output_path):
+        cmd(archiver, "extract", "dst")
+    assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True)
+
+
+def test_export_import_tar_zst_mt(archivers, request, monkeypatch):
+    # exercise the multi-threaded compression path (nb_workers) of the in-process zstd filter.
+    archiver = request.getfixturevalue(archivers)
+    monkeypatch.setenv("BORG_ZSTD_MT_WORKERS", "2")
+    monkeypatch.setattr("borg.compress._zstd_mt_workers", None)  # drop the cache
+    create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
+    os.unlink("input/flagfile")
+    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "create", "src", "input")
+    cmd(archiver, "export-tar", "src", "simple.tar.zst")
+    cmd(archiver, "import-tar", "dst", "simple.tar.zst")
+    with changedir(archiver.output_path):
+        cmd(archiver, "extract", "dst")
+    assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True)
+
+
+@requires_gnutar
+@requires_zstd
+def test_export_tar_zst_interop(archivers, request):
+    # in-process zstd output must be readable by the standard zstd tool.
+    archiver = request.getfixturevalue(archivers)
+    create_test_files(archiver.input_path)
+    os.unlink("input/flagfile")
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input")
+    cmd(archiver, "export-tar", "test", "simple.tar.zst", "--tar-format=GNU")
+    subprocess.check_call(["zstd", "-d", "simple.tar.zst", "-o", "simple.tar"])
+    with changedir("output"):
+        subprocess.check_call(["tar", "xpf", "../simple.tar", "--warning=no-timestamp"])
+    assert_dirs_equal("input", "output/input", ignore_flags=True, ignore_xattrs=True, ignore_ns=True)
+
+
+@requires_zstd
+def test_tar_filter_zstd_external(archivers, request):
+    # an explicit --tar-filter always runs the external filter program, also for zstd.
+    archiver = request.getfixturevalue(archivers)
+    create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
+    os.unlink("input/flagfile")
+    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "create", "src", "input")
+    cmd(archiver, "export-tar", "src", "simple.tar.zst", "--tar-filter=zstd")
+    with open("simple.tar.zst", "rb") as fd:
+        assert fd.read(4) == ZSTD_MAGIC
+    cmd(archiver, "import-tar", "dst", "simple.tar.zst", "--tar-filter=zstd -d")
+    with changedir(archiver.output_path):
+        cmd(archiver, "extract", "dst")
+    assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True)
+
+
 @requires_gnutar
 def test_import_concatenated_tar_with_ignore_zeros(archivers, request):
     archiver = request.getfixturevalue(archivers)

From a68c660b0330a084d01c28616883715ba090e169 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 15:12:13 +0200
Subject: [PATCH 085/144] CI: OpenBSD: put TMPDIR on an mfs

The OpenBSD job is the slowest in the vm_tests matrix (~51min vs.
~26min for FreeBSD). Put the temp tree (pip/cc build temps, pytest tmp
dirs and the borg test repos in them) on a memory-backed filesystem
instead of the FFS disk: pytest 37:40 -> 34:14.

- mfs sized 6g: pytest only cleans its tmp tree up after the run; the
  OpenBSD VM gets 12G RAM to back it (new per-OS matrix.memory
  override, the other VMs stay at 8G).
- -O2 (FFS2 format): mount_mfs defaults to FFS1, whose 32 bit
  timestamps silently wrap around instead of failing with EOVERFLOW:
  test_extract_y2261 got 593175808 (= 9183110400 mod 2^32) back from
  utime().

Also fold the four pkg_add invocations into one.
---
 .github/workflows/ci.yml | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 16b66dadda..0381d8c137 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -419,6 +419,8 @@ jobs:
             version: '7.8'
             display_name: OpenBSD
             do_binaries: false
+            # extra RAM for the mfs-backed TMPDIR, see the openbsd test step
+            memory: 12G
 
           - os: omnios
             version: 'r151056'
@@ -444,7 +446,7 @@ jobs:
           shell: bash
           # the default VM size (2 cpus) leaves half of the runner's 4 vcpus idle
           cpu_count: 4
-          memory: 8G
+          memory: ${{ matrix.memory || '8G' }}
 
       - name: Test on ${{ matrix.display_name }}
         shell: cpa.sh {0}
@@ -540,10 +542,18 @@ jobs:
               ;;
 
             openbsd)
-              sudo -E pkg_add lz4 git
-              sudo -E pkg_add rust
-              sudo -E pkg_add openssl%3.5
-              sudo -E pkg_add py3-pip py3-virtualenv py3-tox
+              # Put the temp tree (pip/cc build temps, pytest tmp dirs and the
+              # borg test repos in them) on a memory-backed filesystem instead
+              # of the slow FFS disk. Sized generously (the pytest tmp tree is
+              # only cleaned up after the run); the VM gets 12G RAM for this.
+              sudo mkdir -p /mfs
+              # -O2: FFS2 format - mfs defaults to FFS1, whose 32 bit
+              # timestamps silently wrap (breaks test_extract_y2261).
+              sudo mount_mfs -O2 -s 6g swap /mfs
+              sudo chmod 1777 /mfs
+              export TMPDIR=/mfs
+
+              sudo -E pkg_add lz4 git rust openssl%3.5 py3-pip py3-virtualenv py3-tox
 
               export BORG_OPENSSL_NAME=eopenssl35
               tox -e py312-none

From 9b3c4c590ceab5636fff14c66d480194a19916f8 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 14:23:26 +0200
Subject: [PATCH 086/144] CI: upgrade to OpenBSD 7.9

Image available since cross-platform-actions/openbsd-builder v0.14.0.
7.9's default python3 is 3.13 (was: 3.12), so switch the tox env
accordingly. openssl 3.5 (eopenssl35) is still the packaged version.
---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0381d8c137..75c2f50e9e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -416,7 +416,7 @@ jobs:
             do_binaries: false
 
           - os: openbsd
-            version: '7.8'
+            version: '7.9'
             display_name: OpenBSD
             do_binaries: false
             # extra RAM for the mfs-backed TMPDIR, see the openbsd test step
@@ -556,7 +556,7 @@ jobs:
               sudo -E pkg_add lz4 git rust openssl%3.5 py3-pip py3-virtualenv py3-tox
 
               export BORG_OPENSSL_NAME=eopenssl35
-              tox -e py312-none
+              tox -e py313-none
               ;;
 
             omnios)

From 284ca3b0812a0439e381ec685ff09016acd0aa59 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 14:24:05 +0200
Subject: [PATCH 087/144] CI: cache pip-built wheels for the VM tests

The VMs spend ~11 minutes per run pip-building every dependency from
source (there are no PyPI wheels for the BSDs / OmniOS / Haiku).

Point PIP_CACHE_DIR into the workspace, which cross-platform-actions
rsyncs into the VM and back out, and persist it on the runner with
actions/cache (keyed per OS/version and the locked requirements, like
the existing linux pip cache). pip then reuses the wheels it built
from sdists in earlier runs. tox passes PIP_* through to its envs by
default.
---
 .github/workflows/ci.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 75c2f50e9e..d3fed44fea 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -434,6 +434,17 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
+      # .pip-cache lives inside the workspace, which cross-platform-actions
+      # rsyncs into the VM and back, so wheels built from sdists in one run
+      # (most of the VM setup time) are reused by the next one.
+      - name: Cache pip-built wheels
+        uses: actions/cache@v6
+        with:
+          path: .pip-cache
+          key: ${{ matrix.os }}-${{ matrix.version }}-pip-${{ hashFiles('requirements.d/development.lock.txt') }}
+          restore-keys: |
+            ${{ matrix.os }}-${{ matrix.version }}-pip-
+
       - name: Start VM for ${{ matrix.display_name }}
         id: cross_os
         # a normal boot takes < 1 minute; since 1.4.0 the action bounds
@@ -459,6 +470,9 @@ jobs:
           # job timeout (tox passes all env vars through to pytest).
           export PYTEST_TIMEOUT=300
 
+          # see the "Cache pip-built wheels" step
+          export PIP_CACHE_DIR="$PWD/.pip-cache"
+
           case "${{ matrix.os }}" in
             freebsd)
               export IGNORE_OSVERSION=yes

From 8d9e5e51ccde7b0a4ff73c7d4d994de6f63d7cc4 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 16:27:47 +0200
Subject: [PATCH 088/144] CI: cache pip-built wheels for the Windows jobs

MSYS2's mingw Python cannot use PyPI's win_amd64 wheels, so pip builds
all compiled deps from source on every run - including building maturin
via cargo (~4 min!) just to build the blake3 wheel.

Point PIP_CACHE_DIR into the workspace and persist it with
actions/cache, like the vm_tests jobs already do: kills ~5 min of the
windows_tests Build step on cache hits. Also applied to the (daily)
windows canary job, which shares the cache via the key prefix.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/canary.yml | 15 +++++++++++++++
 .github/workflows/ci.yml     | 14 ++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml
index c36a6d2737..36fcc0a796 100644
--- a/.github/workflows/canary.yml
+++ b/.github/workflows/canary.yml
@@ -110,6 +110,9 @@ jobs:
       PY_COLORS: 1
       MSYS2_ARG_CONV_EXCL: "*"
       MSYS2_ENV_CONV_EXCL: "*"
+      # see the "Cache pip-built wheels" step. MSYS2_ENV_CONV_EXCL above keeps
+      # this a Windows path when it enters the msys2 shell.
+      PIP_CACHE_DIR: ${{ github.workspace }}\.pip-cache
 
     defaults:
       run:
@@ -120,6 +123,18 @@ jobs:
         with:
           fetch-depth: 0
 
+      # Same as the "Cache pip-built wheels" step in ci.yml (MSYS2's mingw
+      # Python cannot use PyPI's win_amd64 wheels). Own key, because the
+      # unlocked requirements may resolve to different versions; falls back
+      # to (and gets picked up by) the ci.yml cache via the shared prefix.
+      - name: Cache pip-built wheels
+        uses: actions/cache@v6
+        with:
+          path: .pip-cache
+          key: windows-msys2-pip-canary-${{ hashFiles('pyproject.toml', 'requirements.d/pyinstaller.txt') }}
+          restore-keys: |
+            windows-msys2-pip-
+
       - uses: msys2/setup-msys2@v2
         with:
           msystem: UCRT64
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d3fed44fea..ae34d02ad5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -671,6 +671,9 @@ jobs:
       PY_COLORS: 1
       MSYS2_ARG_CONV_EXCL: "*"
       MSYS2_ENV_CONV_EXCL: "*"
+      # see the "Cache pip-built wheels" step. MSYS2_ENV_CONV_EXCL above keeps
+      # this a Windows path when it enters the msys2 shell.
+      PIP_CACHE_DIR: ${{ github.workspace }}\.pip-cache
 
     defaults:
       run:
@@ -681,6 +684,17 @@ jobs:
         with:
           fetch-depth: 0
 
+      # MSYS2's mingw Python cannot use PyPI's win_amd64 wheels, so pip builds
+      # all compiled deps from source - incl. building maturin via cargo, just
+      # to build the blake3 wheel. Persist the wheels pip builds.
+      - name: Cache pip-built wheels
+        uses: actions/cache@v6
+        with:
+          path: .pip-cache
+          key: windows-msys2-pip-${{ hashFiles('pyproject.toml', 'requirements.d/pyinstaller.txt') }}
+          restore-keys: |
+            windows-msys2-pip-
+
       - uses: msys2/setup-msys2@v2
         with:
           msystem: UCRT64

From 4f861a38d290ff5a8348ab5bcd8f4fcdf462b74b Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 11 Aug 2026 15:35:22 +0200
Subject: [PATCH 089/144] write_chunkindex_to_repo: reduce memory needs, fixes
 #9886

A full index rewrite used to build one sorted Python list of all keys
(~90 bytes per key, e.g. ~9 GB for a 100M chunks repo). As the keys are
uniformly distributed hash digests, partition them by their leading key
bits into 2 ** prefix_bits similarly sized, disjoint sets and select /
sort / write one partition's keys at a time, so the extra memory needed
is bounded by one partition (~ one fragment, <= 32 MB worth of keys),
independent of the index size.

Selecting a partition is a cheap, C-level filtering scan of the hash
table (borghash 0.2.0 items(prefix_bits=..., prefix=...)), so runtime
stays unchanged (measured: 21s for a 5M-entry full rewrite either way,
peak extra memory 480 MB -> 85 MB).

Partition membership and prefix_bits (chosen from the entry count) only
depend on the selected entries, so identical entry sets still produce
identical fragments (writing/repacking stays idempotent and convergent),
and as the prefix compares the keys' leading bits, ascending prefixes
yield the same globally sorted key sequence as one all-keys sort did.

ChunkIndex gets a new_count property (count of F_NEW entries, maintained
incrementally) so the incremental write path can pick prefix_bits
without an extra counting pass; as a bonus, a nothing-new incremental
write now skips the full table scan entirely.

Requires borghash ~= 0.2.0.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 pyproject.toml                       |  2 +-
 src/borg/cache.py                    | 49 +++++++++++++++----
 src/borg/hashindex.pyi               |  5 +-
 src/borg/hashindex.pyx               | 35 ++++++++++++--
 src/borg/testsuite/cache_test.py     | 58 +++++++++++++++++++++++
 src/borg/testsuite/hashindex_test.py | 70 ++++++++++++++++++++++++++++
 6 files changed, 204 insertions(+), 15 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 74322d99e0..9a83f254ee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,7 +30,7 @@ classifiers = [
 license = "BSD-3-Clause"
 license-files = ["LICENSE", "AUTHORS"]
 dependencies = [
-  "borghash ~= 0.1.0",
+  "borghash ~= 0.2.0",
   "borgstore[rest,blake3] ~= 0.6.0",
   "msgpack >=1.0.3, <=1.2.1",
   "packaging",
diff --git a/src/borg/cache.py b/src/borg/cache.py
index 0f539adbb3..0717f87c72 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -662,16 +662,45 @@ def write_chunkindex_to_repo(
     new_hashes = set()  # content hashes of the fragments that make up the index we are writing now
     fragments_written = 0
 
-    # sort the selected keys, so that an identical set of entries always produces identical
-    # fragments (identical content hashes), no matter in which order the entries were inserted
-    # into the hash table. this makes writing/repacking idempotent and convergent across clients:
-    # a fragment that already exists in the repo is not stored again (see _store_chunkindex_fragment)
-    # and no differently-partitioned duplicates of the same entries can pile up.
-    keys = sorted(key for key, _ in chunks.iteritems(only_new=incremental))
-    total = len(keys)
-    if keys:
-        # partition the selected entries into batches of at most max_entries entries:
-        batches = chunkit(keys, max_entries)
+    total = chunks.new_count if incremental else len(chunks)
+    if total > max_entries:
+        # to keep memory usage low, we don't build one huge, sorted list of all selected keys
+        # (~90 bytes per key!), see #9886: as the keys are uniformly distributed hash digests, we can
+        # partition them into 2 ** prefix_bits similarly sized, disjoint sets by their leading key bits
+        # and select / sort / write one partition's keys at a time. selecting a partition is a cheap,
+        # C-level filtering scan of the in-memory hash table, see borghash.HashTable.items.
+        # aim a bit below max_entries: without that headroom, a total of exactly
+        # 2 ** prefix_bits * max_entries would put the expected partition size right AT max_entries,
+        # so about half the partitions would overshoot it by a little.
+        target_entries = max_entries - max_entries // 20  # 5% headroom
+        partitions = -(-total // target_entries)  # == ceil(total / target_entries)
+        prefix_bits = (partitions - 1).bit_length()  # smallest prefix_bits with 2 ** prefix_bits >= partitions
+    else:
+        prefix_bits = 0  # all selected keys are one partition (and no filtering scans are needed)
+
+    def gen_batches():
+        # sort the selected keys per partition, so that an identical set of entries always produces
+        # identical fragments (identical content hashes), no matter in which order the entries were
+        # inserted into the hash table: partition membership and prefix_bits (chosen by entry count)
+        # only depend on the selected entries. this makes writing/repacking idempotent and convergent
+        # across clients: a fragment that already exists in the repo is not stored again (see
+        # _store_chunkindex_fragment) and no differently-partitioned duplicates of the same entries
+        # can pile up. as the prefix compares the keys' leading bits, ascending prefixes yield the
+        # same globally sorted key sequence a single all-keys sort would have produced.
+        for prefix in range(2**prefix_bits):
+            keys = sorted(
+                key for key, _ in chunks.iteritems(only_new=incremental, prefix_bits=prefix_bits, prefix=prefix)
+            )
+            # Usually a no-op splitwise: with hash digests as keys, prefix_bits was chosen so that a
+            # partition comes out well below max_entries (>100 sigma of margin), so this yields the
+            # partition as a single batch. It is what makes the "no fragment has more than
+            # max_entries entries" invariant a hard guarantee rather than an assumption about the
+            # key distribution - keys that are not uniformly distributed (e.g. in tests) can put
+            # everything into one partition.
+            yield from chunkit(keys, max_entries)
+
+    if total:
+        batches = gen_batches()
     elif force_write:
         # write a single empty fragment (e.g. at repo creation or after delete_chunkindex_from_repo()):
         batches = [[]]
diff --git a/src/borg/hashindex.pyi b/src/borg/hashindex.pyi
index 16795b7ccd..8a9c43d8b3 100644
--- a/src/borg/hashindex.pyi
+++ b/src/borg/hashindex.pyi
@@ -32,11 +32,14 @@ class ChunkIndex:
     M_SYSTEM: int
     def add(self, key: bytes, size: int) -> None: ...
     def update_pack_info(self, pack_results: list | None) -> None: ...
-    def iteritems(self, *, only_new: bool = ...) -> Iterator: ...
+    def iteritems(self, *, only_new: bool = ..., prefix_bits: int = ..., prefix: int = ...) -> Iterator: ...
+    @property
+    def new_count(self) -> int: ...
     def clear_new(self) -> None: ...
     def __contains__(self, key: bytes) -> bool: ...
     def __getitem__(self, key: bytes) -> type[ChunkIndexEntry]: ...
     def __setitem__(self, key: bytes, value: CIE) -> None: ...
+    def __delitem__(self, key: bytes) -> None: ...
 
 class FuseVersionsIndexEntry(NamedTuple):
     version: int
diff --git a/src/borg/hashindex.pyx b/src/borg/hashindex.pyx
index f903a282fc..a8764b724c 100644
--- a/src/borg/hashindex.pyx
+++ b/src/borg/hashindex.pyx
@@ -59,19 +59,34 @@ class ChunkIndex(HTProxyMixin, MutableMapping):
     def __init__(self, capacity=1000, path=None, usable=None):
         if path:
             self.ht = HashTableNT.read(path)
+            self._new_count = None  # unknown, computed lazily (see new_count)
         else:
             if usable is not None:
                 capacity = usable * 2  # load factor 0.5
             self.ht = HashTableNT(key_size=32, value_type=ChunkIndexEntry, value_format=ChunkIndexEntryFormat,
                                   capacity=capacity)
+            self._new_count = 0
 
     def hide_system_flags(self, value):
         user_flags = value.flags & self.M_USER
         return value._replace(flags=user_flags)
 
-    def iteritems(self, *, only_new=False):
-        """Iterates items (optionally only new items); hides system flags."""
-        for key, value in self.ht.items():
+    @property
+    def new_count(self):
+        """Count of new entries (entries that have the F_NEW flag set)."""
+        if self._new_count is None:
+            # loaded from a file: compute once, then maintain incrementally.
+            self._new_count = sum(1 for key, value in self.ht.items() if value.flags & self.F_NEW)
+        return self._new_count
+
+    def iteritems(self, *, only_new=False, prefix_bits=0, prefix=0):
+        """
+        Iterates items (optionally only new items and/or only items of one key-prefix partition);
+        hides system flags.
+
+        See borghash.HashTable.items for the prefix_bits / prefix semantics.
+        """
+        for key, value in self.ht.items(prefix_bits=prefix_bits, prefix=prefix):
             if not only_new or (value.flags & self.F_NEW):
                 yield key, self.hide_system_flags(value)
 
@@ -101,9 +116,11 @@ class ChunkIndex(HTProxyMixin, MutableMapping):
         except KeyError:
             prev_flags = self.F_NONE
             is_new = True
+            inserting = True
         else:
             prev_flags = prev.flags
             is_new = bool(prev_flags & self.F_NEW)  # was new? stays new!
+            inserting = False
         system_flags = prev_flags & self.M_SYSTEM
         if is_new:
             system_flags |= self.F_NEW
@@ -111,6 +128,17 @@ class ChunkIndex(HTProxyMixin, MutableMapping):
             system_flags &= ~self.F_NEW
         user_flags = value.flags & self.M_USER
         self.ht[key] = value._replace(flags=system_flags | user_flags)
+        if inserting and self._new_count is not None:
+            self._new_count += 1  # inserted a new entry (overwriting keeps the F_NEW state)
+
+    def __delitem__(self, key):
+        value = self.ht.pop(key)  # raises KeyError if not present, like del would
+        if self._new_count is not None and (value.flags & self.F_NEW):
+            self._new_count -= 1
+
+    def clear(self):
+        self.ht.clear()
+        self._new_count = 0
 
     def update_pack_info(self, pack_results):
         """Set pack_id, obj_offset and obj_size from a list of (chunk_id, pack_id, obj_offset, obj_size)
@@ -133,6 +161,7 @@ class ChunkIndex(HTProxyMixin, MutableMapping):
             if value.flags & self.F_NEW:
                 flags = value.flags & ~self.F_NEW
                 self.ht[key] = value._replace(flags=flags)
+        self._new_count = 0
 
     @classmethod
     def read(cls, path):
diff --git a/src/borg/testsuite/cache_test.py b/src/borg/testsuite/cache_test.py
index 8e43e1cdec..b82e6b45bc 100644
--- a/src/borg/testsuite/cache_test.py
+++ b/src/borg/testsuite/cache_test.py
@@ -1,3 +1,4 @@
+import hashlib
 import os
 import time
 from datetime import UTC, datetime
@@ -379,6 +380,63 @@ def test_write_chunkindex_deterministic_fragments(tmp_path, monkeypatch):
     assert hashes[0] == hashes[1]  # identical fragment sets from differently-ordered inputs
 
 
+def _u_key(i):
+    """A pseudo-random (uniformly distributed) 32-byte chunk id for entry number i."""
+    return hashlib.sha256(i.to_bytes(4, "big")).digest()
+
+
+def test_write_chunkindex_partitioned_write(tmp_path, monkeypatch):
+    """A big write with uniformly distributed keys is partitioned by leading key bits.
+
+    When more than MAX entries are selected, the write path processes the keys one key-prefix
+    partition at a time instead of building one huge all-keys list (#9886). The resulting
+    fragments must still be bounded by MAX and together hold every entry exactly once; as the
+    partitions compare the keys' leading bits, the fragments are contiguous ranges of the
+    globally sorted key space.
+    """
+    monkeypatch.setattr(cache_mod, "CHUNKINDEX_FRAGMENT_ENTRIES_MAX", 100)
+    monkeypatch.setattr(cache_mod, "CHUNKINDEX_FRAGMENT_ENTRIES_MIN", 50)
+
+    keys = [_u_key(i) for i in range(1000)]
+    for incremental in (False, True):
+        repository_location = os.fspath(tmp_path / f"repository{incremental}")
+        with Repository(repository_location, exclusive=True, create=True) as repository:
+            delete_chunkindex_from_repo(repository)  # start from a known-empty fragment set
+            write_chunkindex_to_repo(
+                repository, _make_chunkindex(keys), incremental=incremental, force_write=not incremental
+            )
+            fragment_keys = [
+                sorted(read_chunkindex_from_repo(repository, name)) for name, _ in list_chunkindex_fragments(repository)
+            ]
+            # 1000 uniformly distributed entries with MAX=100 must have been really partitioned:
+            assert len(fragment_keys) >= 10
+            assert all(len(fk) <= 100 for fk in fragment_keys)
+            # ordering the fragments by their first key must yield the globally sorted key sequence,
+            # so the fragments are disjoint, complete and contiguous:
+            fragment_keys.sort(key=lambda fk: fk[0])
+            assert [k for fk in fragment_keys for k in fk] == sorted(keys)
+            # the index rebuilt from the fragments is complete:
+            chunks = build_chunkindex_from_repo(repository)
+            assert len(chunks) == 1000
+            assert set(chunks) == set(keys)
+
+
+def test_write_chunkindex_partitioned_deterministic(tmp_path, monkeypatch):
+    """Partitioned writes are convergent, too: identical entries -> identical fragments."""
+    monkeypatch.setattr(cache_mod, "CHUNKINDEX_FRAGMENT_ENTRIES_MAX", 100)
+
+    keys = [_u_key(i) for i in range(1000)]
+    shuffled = keys[311:] + keys[:311]  # same set, different insertion order
+    hashes = []
+    for key_list in (keys, shuffled):
+        repository_location = os.fspath(tmp_path / f"repository{len(hashes)}")
+        with Repository(repository_location, exclusive=True, create=True) as repository:
+            delete_chunkindex_from_repo(repository)
+            write_chunkindex_to_repo(repository, _make_chunkindex(key_list), incremental=False, force_write=True)
+            hashes.append({name for name, _ in list_chunkindex_fragments(repository)})
+    assert hashes[0] == hashes[1]  # identical fragment sets from differently-ordered inputs
+
+
 def test_close_consolidates_fragments_across_sessions(tmp_path, monkeypatch):
     """End-to-end: repeated create-like sessions leave a bounded, consolidated set of fragments."""
     monkeypatch.setenv("BORG_PASSPHRASE", "test")
diff --git a/src/borg/testsuite/hashindex_test.py b/src/borg/testsuite/hashindex_test.py
index 67e3c29d87..7b56f93c33 100644
--- a/src/borg/testsuite/hashindex_test.py
+++ b/src/borg/testsuite/hashindex_test.py
@@ -92,3 +92,73 @@ def new_chunks():
     assert new_chunks() == [(key2, value2a)]
     chunks.clear_new()
     assert new_chunks() == []
+
+
+def _cie(key):
+    return ChunkIndexEntry(flags=ChunkIndex.F_USED, size=1, pack_id=key, obj_offset=0, obj_size=0)
+
+
+def test_new_count():
+    chunks = ChunkIndex()
+    assert chunks.new_count == 0
+    keys = [H2(x) for x in range(10)]
+    for i, key in enumerate(keys):
+        chunks[key] = _cie(key)
+        assert chunks.new_count == i + 1
+    chunks[keys[0]] = _cie(keys[0])  # overwriting a new entry: it stays new, not counted twice
+    assert chunks.new_count == 10
+    del chunks[keys[9]]  # deleting a new entry decrements
+    assert chunks.new_count == 9
+    chunks.clear_new()
+    assert chunks.new_count == 0
+    chunks[keys[9]] = _cie(keys[9])  # re-inserting: new again
+    assert chunks.new_count == 1
+    chunks[keys[0]] = _cie(keys[0])  # overwriting a not-new entry: it stays not-new
+    assert chunks.new_count == 1
+    del chunks[keys[0]]  # deleting a not-new entry: no change
+    assert chunks.new_count == 1
+    with pytest.raises(struct.error):  # a failing insert must not bump the count
+        chunks[H2(11)] = ChunkIndexEntry(flags=ChunkIndex.F_NONE, size=2**33, pack_id=H2(11), obj_offset=0, obj_size=0)
+    assert chunks.new_count == 1
+    chunks.clear()
+    assert chunks.new_count == 0
+
+
+def test_new_count_after_read(tmp_path):
+    # .write() persists the raw entries including the F_NEW flag, so a freshly loaded
+    # ChunkIndex must compute new_count (lazily) from the loaded entries.
+    chunks = ChunkIndex()
+    keys = [H2(x) for x in range(5)]
+    for key in keys:
+        chunks[key] = _cie(key)
+    chunks.clear_new()
+    new_key = H2(1000)
+    chunks[new_key] = _cie(new_key)
+    path = str(tmp_path / "chunks")
+    chunks.write(path)
+    loaded = ChunkIndex.read(path)
+    assert loaded.new_count == 1
+    loaded.clear_new()
+    assert loaded.new_count == 0
+
+
+def test_iteritems_prefix():
+    prefix_bits = 3
+    chunks = ChunkIndex()
+    keys = [H2(x) for x in range(100)]
+    for key in keys:
+        chunks[key] = _cie(key)
+    collected = []
+    for prefix in range(2**prefix_bits):
+        part = [key for key, _ in chunks.iteritems(prefix_bits=prefix_bits, prefix=prefix)]
+        assert all(key[0] >> (8 - prefix_bits) == prefix for key in part)
+        collected += part
+    assert sorted(collected) == sorted(keys)  # complete and disjoint
+    # combining a prefix filter with only_new:
+    chunks.clear_new()
+    new_key = H2(1000)
+    chunks[new_key] = _cie(new_key)
+    new_prefix = new_key[0] >> (8 - prefix_bits)
+    for prefix in range(2**prefix_bits):
+        part = [key for key, _ in chunks.iteritems(only_new=True, prefix_bits=prefix_bits, prefix=prefix)]
+        assert part == ([new_key] if prefix == new_prefix else [])

From acf222a9795c70886afe3a8eadd72a1ab0355464 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 00:49:25 +0200
Subject: [PATCH 090/144] add ProgressIndicatorSpinner, for work of unknown
 duration

The other progress indicators either count (percent) or just emit a message.
For work whose duration or size is not known in advance, neither fits: what we
want to show is "borg is still working, at this", and that is a spinner.

It is pull-based on purpose: no thread, no timer, nothing runs while the caller
is not calling. show() is rate limited by BORG_PROGRESS_FPS, so calling it very
often is cheap.

On a terminal, the frame and the message are repainted in place, in borg green.
Without a terminal (or with --log-json) there is nothing to animate, so only
message changes are logged, as "progress_message" - same as
ProgressIndicatorMessage does it, so frontends need no changes.

The frames are all East Asian Width "Neutral", so they take exactly one cell
and the message never shifts column. If the output encoding can not represent
them (or BORG_SPINNER=ascii), the classic ASCII frames are used instead.
BORG_SPINNER=off never animates.
---
 docs/usage/general/environment.rst.inc      |  12 ++
 src/borg/archiver/help_cmd.py               |  12 ++
 src/borg/helpers/__init__.py                |   2 +-
 src/borg/helpers/progress.py                | 180 +++++++++++++++++-
 src/borg/testsuite/helpers/progress_test.py | 201 +++++++++++++++++++-
 5 files changed, 404 insertions(+), 3 deletions(-)

diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc
index 5132f4e53b..d9ac6db6db 100644
--- a/docs/usage/general/environment.rst.inc
+++ b/docs/usage/general/environment.rst.inc
@@ -95,6 +95,18 @@ General:
         ``BORG_PROGRESS_FPS=0.1`` limits it to one update every 10 seconds.
         Lower values are useful when the output goes into a logfile rather than
         to an interactive terminal.
+    BORG_SPINNER
+        Controls the spinner borg animates on a terminal while doing work of unknown
+        duration:
+
+        - unset (default): animate, using Unicode frames if the terminal can display them
+        - ``ascii``: animate, but only use ASCII frames (``|/-\``)
+        - ``off``: do not animate, only output the messages next to the spinner
+
+        The spinner is animated only on an interactive terminal anyway (and never
+        with ``--log-json``), and its colour follows the usual ``NO_COLOR`` and
+        ``COLORTERM`` conventions. See also ``BORG_PROGRESS_FPS``: it also gives
+        the spinner its frame rate.
     BORG_DEBUG_PROFILE
         When set to a filename, write an execution profile in Borg format into that file
         (see :ref:`debugging`). If the filename ends with ``.pyprof``, a Python-compatible
diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py
index 296eeb126e..6ae5e40b09 100644
--- a/src/borg/archiver/help_cmd.py
+++ b/src/borg/archiver/help_cmd.py
@@ -664,6 +664,18 @@ class HelpMixIn:
                 ``BORG_PROGRESS_FPS=0.1`` limits it to one update every 10 seconds.
                 Lower values are useful when the output goes into a logfile rather than
                 to an interactive terminal.
+            BORG_SPINNER
+                Controls the spinner borg animates on a terminal while doing work of unknown
+                duration:
+
+                - unset (default): animate, using Unicode frames if the terminal can display them
+                - ``ascii``: animate, but only use ASCII frames (``|/-\\``)
+                - ``off``: do not animate, only output the messages next to the spinner
+
+                The spinner is animated only on an interactive terminal anyway (and never
+                with ``--log-json``), and its colour follows the usual ``NO_COLOR`` and
+                ``COLORTERM`` conventions. See also ``BORG_PROGRESS_FPS``: it also gives
+                the spinner its frame rate.
             BORG_DEBUG_PROFILE
                 When set to a filename, write an execution profile in Borg format into that file
                 (see :ref:`debugging`). If the filename ends with ``.pyprof``, a Python-compatible
diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py
index 703e2d839c..21d2e764c2 100644
--- a/src/borg/helpers/__init__.py
+++ b/src/borg/helpers/__init__.py
@@ -53,7 +53,7 @@
 from .process import daemonize, daemonizing, ThreadRunner
 from .process import signal_handler, raising_signal_handler, sig_int, ignore_sigint, SigHup, SigTerm
 from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process
-from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage, get_progress_dt
+from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage, ProgressIndicatorSpinner, get_progress_dt
 from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS
 from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now
 from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH
diff --git a/src/borg/helpers/progress.py b/src/borg/helpers/progress.py
index c781dbae8a..87aa04174a 100644
--- a/src/borg/helpers/progress.py
+++ b/src/borg/helpers/progress.py
@@ -1,9 +1,12 @@
 import logging
 import json
 import os
+import sys
 import time
+from shutil import get_terminal_size
 
-from ..logger import create_logger
+from .parseformat import ellipsis_truncate
+from ..logger import create_logger, JSONProgressFormatter
 
 logger = create_logger()
 
@@ -109,3 +112,178 @@ def show(self, current=None, increase=1, info=None):
     def output(self, message, info=None):
         j = self.make_json(message=message, current=self.counter, total=self.total, info=info)
         self.logger.info(j)
+
+
+# borg green, as on borgbackup.org: #22D045
+GREEN_TRUECOLOR = "\x1b[38;2;34;208;69m"
+# The nearest xterm-256 entry (41 == #00d75f) drops the red channel and reads cooler and more
+# acid than the brand green, so we rather use the bright green from the terminal's own palette:
+# it behaves better on light backgrounds and on other people's terminals.
+GREEN_16 = "\x1b[92m"
+ANSI_RESET = "\x1b[0m"
+ANSI_HIDE_CURSOR = "\x1b[?25l"
+ANSI_SHOW_CURSOR = "\x1b[?25h"
+ANSI_CLEAR_LINE = "\r\x1b[2K"  # carriage return, then erase the whole line
+
+# All frames have East Asian Width "Neutral", so they occupy exactly one cell each and the
+# message after the spinner never shifts column while the animation runs.
+SPINNER_FRAMES = {
+    "square": "▫▪◻◼",  # ▫ ▪ ◻ ◼ - pulsing: small to medium, hollow to solid
+    "cube": "◰◳◲◱",  # ◰ ◳ ◲ ◱ - quadrant rotating clockwise
+    "boxed": "⊞⊠⊟⊡",  # ⊞ ⊠ ⊟ ⊡ - squared plus/times/minus/dot
+    "ascii": "|/-\\",  # the classic, for terminals that can not encode the above
+}
+DEFAULT_SPINNER = "square"
+
+
+def _encodable(text, stream):
+    """Can <text> be represented in the encoding of <stream>?"""
+    try:
+        text.encode(getattr(stream, "encoding", None) or "ascii")
+    except (UnicodeEncodeError, LookupError):
+        return False
+    return True
+
+
+def _progress_output_is_json():
+    """Is progress output currently rendered as JSON (--log-json)?"""
+    progress_logger = logging.getLogger(ProgressIndicatorBase.LOGGER)
+    return any(isinstance(handler.formatter, JSONProgressFormatter) for handler in progress_logger.handlers)
+
+
+def spinner_colour(stream):
+    """The ANSI sequence for borg green, or "" if colour is unwanted on <stream>."""
+    if os.environ.get("NO_COLOR") is not None:
+        return ""
+    if os.environ.get("COLORTERM", "").lower() in ("truecolor", "24bit"):
+        return GREEN_TRUECOLOR
+    return GREEN_16
+
+
+class ProgressIndicatorSpinner(ProgressIndicatorBase):
+    """
+    Progress indicator for operations of unknown duration or unknown size.
+
+    Nothing is counted here, the spinner just shows that borg is still working (and, via the
+    message, at what). It is pull-based on purpose: no thread, no timer, nothing runs while the
+    caller is not calling. show() is rate limited to BORG_PROGRESS_FPS internally, so calling it
+    very often is cheap: it costs one time.monotonic() per call and repaints at most that often.
+
+    On a terminal, the spinner frame and the message are repainted in place, in borg green.
+    Without a terminal (or with --log-json), an animation is pointless, so nothing is repainted:
+    only message changes are logged, in the same way ProgressIndicatorMessage does it.
+
+    Usage:
+
+        with ProgressIndicatorSpinner("Deduplicating", msgid="create") as spinner:
+            for chunk in chunks:
+                process(chunk)
+                spinner.show()
+
+    Without the context manager, call finish() yourself - it clears the spinner line and
+    switches the terminal's cursor back on.
+
+    Set BORG_SPINNER=off to never animate, BORG_SPINNER=ascii to force the ASCII frames.
+    """
+
+    JSON_TYPE = "progress_message"  # same contract as ProgressIndicatorMessage, see frontends docs
+
+    def __init__(self, message="", *, frames=DEFAULT_SPINNER, msgid=None, stream=None, animate=None):
+        """
+        :param message: text shown next to the spinner, can be updated via show(message).
+        :param frames: name of a SPINNER_FRAMES set, or a string of frame characters.
+        :param msgid: see ProgressIndicatorBase.
+        :param stream: where to paint the animation [sys.stderr, looked up when writing].
+        :param animate: force the animation on or off [None: decide by looking at the output].
+        """
+        super().__init__(msgid=msgid)
+        self.message = message
+        self._stream = stream
+        mode = os.environ.get("BORG_SPINNER", "").lower()
+        frames = SPINNER_FRAMES.get(frames, frames)
+        self.frames = SPINNER_FRAMES["ascii"] if mode == "ascii" or not _encodable(frames, self.stream) else frames
+        self.animate = self._animation_wanted(mode) if animate is None else animate
+        self.colour = spinner_colour(self.stream) if self.animate else ""
+        self.reset = ANSI_RESET if self.colour else ""
+        self.index = 0  # index of the next frame to paint
+        self.next_update = 0.0  # time.monotonic() value from which on the next output is due
+        self.painted = False  # is there a spinner line on the terminal right now?
+        self.started = False  # did we output anything yet?
+
+    @property
+    def stream(self):
+        """Like logger.StderrHandler, use whatever sys.stderr is *now*, not at construction time."""
+        return sys.stderr if self._stream is None else self._stream
+
+    def _animation_wanted(self, mode):
+        if mode == "off":
+            return False
+        if os.environ.get("TERM", "") in ("", "dumb"):
+            return False
+        try:
+            if not self.stream.isatty():
+                return False
+        except (OSError, ValueError):  # closed or otherwise unusable stream
+            return False
+        return not _progress_output_is_json()
+
+    def show(self, message=None):
+        """Advance the spinner, optionally updating the message. Cheap to over-call."""
+        changed = message is not None and message != self.message
+        if message is not None:
+            self.message = message
+        now = time.monotonic()
+        if now < self.next_update:
+            return
+        if self.animate:
+            self.next_update = now + get_progress_dt()
+            self.paint()
+        elif changed or not self.started:
+            # no terminal to animate on: only say something if there is something new to say.
+            self.next_update = now + get_progress_dt()
+            self.started = True
+            self.output(self.message)
+
+    def paint(self):
+        """Repaint the spinner line in place."""
+        if not self.logger.isEnabledFor(logging.INFO):  # e.g. --quiet
+            return
+        frame = self.frames[self.index % len(self.frames)]
+        self.index += 1
+        # a wrapped line could not be repainted in place, thus the truncation.
+        # ellipsis_truncate pads short messages to the full width - we erase the line instead.
+        columns, lines = get_terminal_size()
+        message = ellipsis_truncate(self.message, columns - 2).rstrip(" ") if self.message else ""
+        text = f"{self.colour}{frame}{self.reset} {message}" if message else f"{self.colour}{frame}{self.reset}"
+        self.write(("" if self.painted else ANSI_HIDE_CURSOR) + ANSI_CLEAR_LINE + text)
+        self.painted = True
+        self.started = True
+
+    def write(self, text):
+        try:
+            self.stream.write(text)
+            self.stream.flush()
+        except (OSError, ValueError):  # the stream went away, e.g. a closed pipe
+            self.animate = False
+
+    def output(self, message):
+        j = self.make_json(message=message)
+        self.logger.info(j)
+
+    def finish(self):
+        """Remove the spinner line from the terminal / log that the operation is finished."""
+        if self.animate:
+            if self.painted:
+                self.write(ANSI_CLEAR_LINE + ANSI_SHOW_CURSOR)
+                self.painted = False
+            # no JSON "finished" record here: if we animate, nobody is parsing our output anyway
+            # and the empty line the text formatter would emit for it is just in the way.
+        else:
+            super().finish()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.finish()
+        return False
diff --git a/src/borg/testsuite/helpers/progress_test.py b/src/borg/testsuite/helpers/progress_test.py
index 40713d1c7d..5bfb7620cb 100644
--- a/src/borg/testsuite/helpers/progress_test.py
+++ b/src/borg/testsuite/helpers/progress_test.py
@@ -1,7 +1,12 @@
+import io
+import logging
+
 import pytest
 
 from ...helpers import progress
-from ...helpers.progress import ProgressIndicatorPercent, get_progress_dt
+from ...helpers.progress import ProgressIndicatorBase, ProgressIndicatorPercent, ProgressIndicatorSpinner
+from ...helpers.progress import get_progress_dt
+from ...helpers.progress import ANSI_CLEAR_LINE, ANSI_HIDE_CURSOR, ANSI_SHOW_CURSOR, GREEN_16, GREEN_TRUECOLOR
 
 
 @pytest.mark.parametrize(
@@ -74,3 +79,197 @@ def test_progress_percentage_quiet(capfd):
     pi.finish()
     out, err = capfd.readouterr()
     assert err == ""
+
+
+class FakeStream(io.StringIO):
+    """A stream that has an encoding, like the real stderr has."""
+
+    encoding = "utf-8"
+
+
+class FakeTTY(FakeStream):
+    """A stream that claims to be a terminal, so the spinner animates into it."""
+
+    def isatty(self):
+        return True
+
+
+@pytest.fixture
+def progress_logger():
+    """The progress logger at the level setup_logging() gives it, restored afterwards."""
+    logger = logging.getLogger(ProgressIndicatorBase.LOGGER)
+    level = logger.level
+    logger.setLevel(logging.INFO)  # setLevel (not .level = ...): it also invalidates the isEnabledFor cache
+    yield logger
+    logger.setLevel(level)
+
+
+@pytest.fixture
+def tty(monkeypatch, progress_logger):
+    monkeypatch.setenv("TERM", "xterm")
+    monkeypatch.setenv("COLUMNS", "80")  # make the terminal width deterministic
+    monkeypatch.setenv("NO_COLOR", "1")  # ... and the escape sequences short
+    monkeypatch.delenv("BORG_SPINNER", raising=False)
+    return FakeTTY()
+
+
+def show_force(spinner, message=None):
+    # bypass the BORG_PROGRESS_FPS rate limiting, so every call produces output
+    spinner.next_update = 0.0
+    spinner.show(message)
+
+
+def test_spinner_animates(tty):
+    spinner = ProgressIndicatorSpinner("Deduplicating", stream=tty)
+    assert spinner.animate
+    show_force(spinner)
+    assert tty.getvalue() == ANSI_HIDE_CURSOR + ANSI_CLEAR_LINE + "▫ Deduplicating"
+    tty.truncate(0), tty.seek(0)
+    show_force(spinner, "Compacting")
+    show_force(spinner)
+    show_force(spinner)
+    show_force(spinner)  # frames wrap around to the first one
+    assert tty.getvalue() == "".join(ANSI_CLEAR_LINE + f"{frame} Compacting" for frame in "▪◻◼▫")
+    tty.truncate(0), tty.seek(0)
+    spinner.finish()
+    assert tty.getvalue() == ANSI_CLEAR_LINE + ANSI_SHOW_CURSOR
+
+
+def test_spinner_rate_limited(tty, monkeypatch):
+    monkeypatch.setenv("BORG_PROGRESS_FPS", "0.1")  # one update per 10s
+    spinner = ProgressIndicatorSpinner("Working", stream=tty)
+    spinner.show()  # the first update is always shown
+    assert tty.getvalue() != ""
+    tty.truncate(0), tty.seek(0)
+    spinner.show()  # immediately after: suppressed by the rate limit
+    spinner.show("even a new message has to wait")
+    assert tty.getvalue() == ""
+
+
+def test_spinner_context_manager(tty):
+    with ProgressIndicatorSpinner("Working", stream=tty) as spinner:
+        show_force(spinner)
+    assert tty.getvalue().startswith(ANSI_HIDE_CURSOR)
+    assert tty.getvalue().endswith(ANSI_CLEAR_LINE + ANSI_SHOW_CURSOR)
+
+
+def test_spinner_nothing_painted_nothing_to_clean_up(tty):
+    with ProgressIndicatorSpinner("Working", stream=tty):
+        pass  # no show() call, so we never hid the cursor and must not paint anything either
+    assert tty.getvalue() == ""
+
+
+def test_spinner_no_message(tty):
+    spinner = ProgressIndicatorSpinner(stream=tty)
+    show_force(spinner)
+    assert tty.getvalue() == ANSI_HIDE_CURSOR + ANSI_CLEAR_LINE + "▫"  # no trailing blank
+
+
+def test_spinner_long_message_truncated(tty):
+    spinner = ProgressIndicatorSpinner(stream=tty)
+    show_force(spinner, "x" * 200)
+    line = tty.getvalue().rsplit(ANSI_CLEAR_LINE, 1)[1]
+    assert len(line) <= 80
+    assert line.startswith("▫ xxx") and line.endswith("xxx")
+
+
+@pytest.mark.parametrize("frames,expected", [("cube", "◰"), ("square", "▫"), ("boxed", "⊞"), ("+x", "+")])
+def test_spinner_frame_sets(tty, frames, expected):
+    spinner = ProgressIndicatorSpinner(frames=frames, stream=tty)
+    show_force(spinner)
+    assert tty.getvalue().endswith(expected)
+
+
+def test_spinner_ascii_forced(tty, monkeypatch):
+    monkeypatch.setenv("BORG_SPINNER", "ascii")
+    spinner = ProgressIndicatorSpinner("Working", stream=tty)
+    show_force(spinner)
+    assert tty.getvalue().endswith("| Working")
+
+
+def test_spinner_ascii_fallback_for_unencodable_frames(tty):
+    tty.encoding = "ascii"  # a terminal that can not represent the fancy frames
+    spinner = ProgressIndicatorSpinner("Working", stream=tty)
+    show_force(spinner)
+    assert tty.getvalue().endswith("| Working")
+
+
+@pytest.mark.parametrize(
+    "env,colour",
+    [
+        ({}, GREEN_16),
+        ({"COLORTERM": "truecolor"}, GREEN_TRUECOLOR),
+        ({"COLORTERM": "24bit"}, GREEN_TRUECOLOR),
+        ({"COLORTERM": "8bit"}, GREEN_16),
+        ({"NO_COLOR": "1"}, ""),
+        ({"NO_COLOR": "1", "COLORTERM": "truecolor"}, ""),
+    ],
+)
+def test_spinner_colour(tty, monkeypatch, env, colour):
+    monkeypatch.delenv("NO_COLOR", raising=False)  # the tty fixture sets it
+    for name, value in env.items():
+        monkeypatch.setenv(name, value)
+    spinner = ProgressIndicatorSpinner("Working", stream=tty)
+    show_force(spinner)
+    assert tty.getvalue().endswith(f"{colour}▫{progress.ANSI_RESET if colour else ''} Working")
+
+
+def test_spinner_quiet(tty, progress_logger):
+    spinner = ProgressIndicatorSpinner("Working", stream=tty)
+    progress_logger.setLevel("WARN")  # e.g. --quiet
+    show_force(spinner)
+    assert tty.getvalue() == ""
+
+
+def test_spinner_broken_stream(tty):
+    spinner = ProgressIndicatorSpinner("Working", stream=tty)
+    tty.close()  # e.g. the terminal went away
+    show_force(spinner)
+    assert not spinner.animate  # we gave up on it, no exception
+
+
+@pytest.mark.parametrize("env", [{"BORG_SPINNER": "off"}, {"TERM": "dumb"}, {"TERM": ""}])
+def test_spinner_animation_disabled(tty, monkeypatch, env):
+    for name, value in env.items():
+        monkeypatch.setenv(name, value)
+    spinner = ProgressIndicatorSpinner("Working", stream=tty)
+    assert not spinner.animate
+    show_force(spinner)
+    assert tty.getvalue() == ""  # nothing painted, it goes to the log instead
+
+
+def test_spinner_no_tty(capfd, monkeypatch, progress_logger):
+    monkeypatch.delenv("BORG_SPINNER", raising=False)
+    spinner = ProgressIndicatorSpinner("Saving files cache")  # stderr is not a tty here
+    assert not spinner.animate
+    show_force(spinner)
+    out, err = capfd.readouterr()
+    assert err == "Saving files cache\n"
+    show_force(spinner)  # the message did not change: an animation frame is all we could add
+    show_force(spinner)
+    out, err = capfd.readouterr()
+    assert err == ""
+    show_force(spinner, "Saving chunks cache")
+    out, err = capfd.readouterr()
+    assert err == "Saving chunks cache\n"
+    spinner.finish()
+    out, err = capfd.readouterr()
+    assert err == "\n"  # the "finished" record, like the other progress indicators emit it
+
+
+def test_spinner_no_tty_quiet(capfd, progress_logger):
+    spinner = ProgressIndicatorSpinner("Saving files cache")
+    progress_logger.setLevel("WARN")  # e.g. --quiet
+    show_force(spinner)
+    spinner.finish()
+    out, err = capfd.readouterr()
+    assert err == ""
+
+
+def test_spinner_animate_forced(monkeypatch, progress_logger):
+    stream = FakeStream()  # not a tty at all
+    monkeypatch.setenv("COLUMNS", "80")
+    monkeypatch.setenv("NO_COLOR", "1")
+    spinner = ProgressIndicatorSpinner("Working", stream=stream, animate=True)
+    show_force(spinner)
+    assert stream.getvalue().endswith("▫ Working")

From bf6a449f79197eda70c45755fd953aed2526c376 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 09:40:58 +0200
Subject: [PATCH 091/144] compression: support zstd negative ("fast") levels,
 fixes #9950

zstd levels below 1 trade compression ratio for speed. They are useful:
on data with long repeats (disk/VM images, database files), zstd,-50
reached more than twice lz4's compression ratio at more than twice its
speed. Which of them is worth using depends a lot on the data, so we
accept the full range the level byte can hold rather than cutting it off
somewhere: zstd,-128 .. zstd,22.

The clevel byte is now interpreted per compressor: for zstd it is an
int8_t, for everything else it stays unsigned as before. Levels 1..22
encode to the byte value they always had, so existing repositories keep
their meaning and no repo format change is needed.

The encoding lives in the new CompressorBase.encode_level/decode_level
classmethods (identity by default, overridden by ZSTD), so only zstd is
affected and every other compressor stores exactly the byte it did before.

repo-compress compares the compression settings it wants against the ones
found in the repo, so both sides have to use the same representation:
get_csettings now returns the stored byte. Otherwise every object would be
parsed and recompressed on every run, only to then keep the old bytes.
format_compression_spec decodes the byte before checking it against 255
("level not applicable"), as 255 is a valid zstd level (-1).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UowHQDcnmow3JnYBmaGDpN
---
 docs/changes.rst                              |  2 +
 docs/internals/data-structures.rst            | 17 +++++--
 src/borg/archiver/completion_cmd.py           | 11 ++++-
 src/borg/archiver/help_cmd.py                 |  9 +++-
 src/borg/archiver/repo_compress_cmd.py        | 16 +++++--
 src/borg/compress.pyx                         | 41 +++++++++++++++--
 src/borg/helpers/parseformat.py               |  6 ++-
 .../archiver/repo_compress_cmd_test.py        | 42 +++++++++++++++++
 src/borg/testsuite/compress_test.py           | 46 ++++++++++++++++++-
 9 files changed, 170 insertions(+), 20 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 16ce70d418..319ae5410f 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -198,6 +198,8 @@ New features:
     via OpenSSL, #9987, #10043
   - zero-copy fill and lazy buffer compaction optimizations
   - log the chunker and its scan kernel at debug level
+- compression: support zstd's negative ("fast") levels, ``zstd,-1`` .. ``zstd,-128``, #9950.
+  They trade compression ratio for speed. Compatible with existing repositories.
 - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice
   replacement for `borg mount` in some use cases, #9942
 - mount: expose POSIX ACLs on Linux mounts (not enforced), #1042
diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index 72a4c428a6..4823404b5e 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -115,7 +115,8 @@ Repo object metadata
 Metadata is a MessagePack-encoded (and encrypted/authenticated) dict with:
 
 - ctype (compression type 0..255)
-- clevel (compression level 0..255)
+- clevel (compression level, one byte, interpreted depending on ctype - see
+  :ref:`data-compression`)
 - csize (overall compressed (and maybe obfuscated) data size)
 - psize (only when obfuscated: payload size without the obfuscation trailer)
 - size (uncompressed size of the data)
@@ -1011,19 +1012,25 @@ Compression
 -----------
 
 Borg supports the following compression methods, each identified by a ctype value
-in the range between 0 and 255 (and augmented by a clevel 0..255 value for the
+in the range between 0 and 255 (and augmented by a one-byte clevel value for the
 compression level):
 
 - none (no compression, pass through data 1:1), identified by 0x00
 - lz4 (low compression, but super fast), identified by 0x01
-- zstd (level 1-22 offering a wide range: level 1 is lower compression and high
-  speed, level 22 is higher compression and lower speed) - identified by 0x03
+- zstd (level -128..22 offering a wide range: level 22 is higher compression and lower
+  speed, level 1 is lower compression and high speed, and the negative "fast" levels
+  trade still more compression for still more speed) - identified by 0x03
 - zlib (level 0-9, level 0 is no compression [but still adding zlib overhead],
   level 1 is low, level 9 is high compression), identified by 0x05
 - lzma (level 0-9, level 0 is low, level 9 is high compression), identified
   by 0x02.
 
-The type byte is followed by a byte indicating the compression level.
+The type byte is followed by a byte indicating the compression level. How that byte is
+interpreted depends on the compression type: for zstd it is an ``int8_t``, so that the
+negative levels fit (level -1 is stored as 255, -128 as 128). For all other types it is
+an unsigned byte, with 255 meaning "no level applies" (as for none and lz4). Levels 1..22
+occupy the same byte values either way, so zstd data written by older borg versions keeps
+its meaning.
 
 Speed:  none > lz4 > zlib > lzma, lz4 > zstd
 Compression: lzma > zlib > lz4 > none, zstd > lz4
diff --git a/src/borg/archiver/completion_cmd.py b/src/borg/archiver/completion_cmd.py
index c50287735d..e3d42c5442 100644
--- a/src/borg/archiver/completion_cmd.py
+++ b/src/borg/archiver/completion_cmd.py
@@ -972,7 +972,16 @@ def for_all_shells(fn_name):
         help_choices = " ".join(sorted(help_choices))
 
         # Compression spec choices (static list)
-        comp_spec_choices = ["lz4", "zstd,3", "auto,zstd,10", "zlib,6", "lzma,6", "obfuscate,250,lz4", "none"]
+        comp_spec_choices = [
+            "lz4",
+            "zstd,3",
+            "zstd,-4",
+            "auto,zstd,10",
+            "zlib,6",
+            "lzma,6",
+            "obfuscate,250,lz4",
+            "none",
+        ]
         comp_spec_choices_str = " ".join(comp_spec_choices)
 
         # Chunker params choices (static list)
diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py
index 296eeb126e..f38ec8a920 100644
--- a/src/borg/archiver/help_cmd.py
+++ b/src/borg/archiver/help_cmd.py
@@ -480,8 +480,15 @@ class HelpMixIn:
 
         zstd[,L]
             Use zstd ("zstandard") compression, a modern wide-range algorithm.
-            If you do not explicitly give the compression level L (ranging from 1
+            If you do not explicitly give the compression level L (ranging from -128
             to 22), it will use level 3.
+            Negative levels are zstd's "fast" levels (level -N is what the zstd command
+            line tool calls --fast=N): they give up compression ratio for speed.
+            -1 to -10 is the useful range for general data; going lower only pays off for
+            data with long repeats (disk/VM images, database files), where it stays fast
+            while still finding the big matches.
+            Level 0 selects zstd's default level (3) - for zstd it does not mean
+            "no compression" (use "none" for that), unlike for zlib below.
 
         zlib[,L]
             Use zlib ("gz") compression. Medium speed, medium compression.
diff --git a/src/borg/archiver/repo_compress_cmd.py b/src/borg/archiver/repo_compress_cmd.py
index 6aa749fc45..1a5089a617 100644
--- a/src/borg/archiver/repo_compress_cmd.py
+++ b/src/borg/archiver/repo_compress_cmd.py
@@ -19,14 +19,18 @@
 
 
 def get_csettings(c):
-    """Return the (ctype, clevel, olevel) compression settings a compressor is configured for."""
+    """Return the (ctype, clevel, olevel) compression settings a compressor is configured for.
+
+    clevel is the *stored* level byte, so that it can be compared against the clevel of an
+    existing repo object as-is - see PackRecompressor.transform.
+    """
     if isinstance(c, Auto):
         return get_csettings(c.compressor)
     if isinstance(c, ObfuscateSize):
         ctype, clevel, _ = get_csettings(c.compressor)
         olevel = c.level
         return ctype, clevel, olevel
-    ctype, clevel, olevel = c.ID, c.level, -1
+    ctype, clevel, olevel = c.ID, c.encode_level(c.level), -1
     return ctype, clevel, olevel
 
 
@@ -37,7 +41,11 @@ def format_compression_spec(ctype, clevel, olevel):
             cname = f"{cname}"
             break
     else:
-        cname = f"{ctype}"
+        cname, cls = f"{ctype}", None
+    if cls is not None:
+        # decode before checking for 255 ("level not applicable"): for zstd the byte is an
+        # int8_t, so 255 there is level -1 and has to be shown.
+        clevel = cls.decode_level(clevel)
     clevel = f",{clevel}" if clevel != 255 else ""
     return obfuscation + cname + clevel
 
@@ -228,7 +236,7 @@ def build_parser_repo_compress(self, subparsers, common_parser, mid_common_parse
             type=CompressionSpec,
             default=CompressionSpec("lz4"),
             action=Highlander,
-            help="select compression algorithm, see the output of the " '"borg help compression" command for details.',
+            help='select compression algorithm, see the output of the "borg help compression" command for details.',
         )
 
         subparser.add_argument("-s", "--stats", dest="stats", action="store_true", help="print statistics")
diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx
index 57019d6259..c31014dcb4 100644
--- a/src/borg/compress.pyx
+++ b/src/borg/compress.pyx
@@ -128,8 +128,24 @@ cdef class CompressorBase:
     def detect(cls, data):
         return data and data[0] == cls.ID
 
+    @classmethod
+    def encode_level(cls, level):
+        """Compression level as the compressor uses it -> the single clevel byte we store.
+
+        Compressors that need a different byte encoding (see ZSTD) override this and
+        decode_level. Doing it per compressor rather than globally keeps the stored byte
+        of every other compressor exactly as it always was.
+        """
+        assert 0 <= level <= 255, f"invalid level: {level}"
+        return level
+
+    @classmethod
+    def decode_level(cls, clevel):
+        """The stored clevel byte -> compression level as the compressor uses it."""
+        return clevel
+
     def __init__(self, level=255, legacy_mode=False, **kwargs):
-        assert 0 <= level <= 255
+        self.encode_level(level)  # rejects levels this compressor cannot store
         self.level = level
         self.legacy_mode = legacy_mode  # True: support prefixed ctype/clevel bytes
 
@@ -152,10 +168,10 @@ cdef class CompressorBase:
         """
         if self.legacy_mode:
             # the ID/level prefix concatenation needs bytes (bytes() is a no-op for bytes input)
-            return None, bytes((self.ID, self.level)) + bytes(data)
+            return None, bytes((self.ID, self.encode_level(self.level))) + bytes(data)
         else:
             meta["ctype"] = self.ID
-            meta["clevel"] = self.level
+            meta["clevel"] = self.encode_level(self.level)
             meta["csize"] = len(data)
             return meta, data
 
@@ -384,10 +400,25 @@ class LZMA(DecidingCompressor):
 class ZSTD(DecidingCompressor):
     """
     zstd compression / decompression (python stdlib (python >= 3.14))
+
+    Levels range from -128 to 22. The negative ones are zstd's "fast" levels (level -N is
+    what the zstd cli calls --fast=N): they trade compression ratio for speed.
     """
     ID = 0x03
     name = 'zstd'
 
+    @classmethod
+    def encode_level(cls, level):
+        # We interpret the clevel byte as an int8_t, which is what makes the negative levels
+        # storable at all. Levels 1..22 encode to the very same byte value as before, so
+        # repos written by older borg versions keep their meaning.
+        assert -128 <= level <= 127, f"invalid level: {level}"
+        return level & 0xFF
+
+    @classmethod
+    def decode_level(cls, clevel):
+        return clevel - 256 if clevel >= 128 else clevel
+
     def __init__(self, level=3, legacy_mode=False, **kwargs):
         super().__init__(level=level, legacy_mode=legacy_mode, **kwargs)
         self.level = level
@@ -715,9 +746,9 @@ class Compressor:
     @staticmethod
     def detect(data):
         hdr = bytes(data[:2])  # detect() does not work with memoryview
-        level = hdr[1]  # usually the level, but not for zlib_legacy
+        clevel = hdr[1]  # usually the level byte, but not for zlib_legacy
         for cls in COMPRESSOR_LIST:
             if cls.detect(hdr):
-                return cls, (255 if cls.name == 'zlib_legacy' else level)
+                return cls, (255 if cls.name == 'zlib_legacy' else cls.decode_level(clevel))
         else:
             raise ValueError('No decompressor for this data found: %r.', data[:2])
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index a007e8a5b6..1ff34fb32a 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -227,8 +227,10 @@ def __init__(self, s):
                     level = 3  # default compression level in zstd
                 elif count == 2:
                     level = int(values[1])
-                    if not 1 <= level <= 22:
-                        raise ArgumentTypeError("level must be >= 1 and <= 22")
+                    # negative levels are zstd's "fast" levels, -128 is what the clevel byte
+                    # can still store (it is interpreted as an int8_t for zstd).
+                    if not -128 <= level <= 22:
+                        raise ArgumentTypeError("level must be >= -128 and <= 22")
                 else:
                     raise ArgumentTypeError("too many arguments")
                 self.level = level
diff --git a/src/borg/testsuite/archiver/repo_compress_cmd_test.py b/src/borg/testsuite/archiver/repo_compress_cmd_test.py
index 38397b04cd..739c799202 100644
--- a/src/borg/testsuite/archiver/repo_compress_cmd_test.py
+++ b/src/borg/testsuite/archiver/repo_compress_cmd_test.py
@@ -1,4 +1,5 @@
 import os
+import re
 
 import pytest
 
@@ -79,6 +80,47 @@ def check_compression(ctype, clevel, olevel):
     cmd(archiver, "check")
 
 
+def test_repo_compress_zstd_negative_level(archiver):
+    """zstd's negative ("fast") levels survive a repo-compress round trip.
+
+    The second run is the interesting part: it only reports everything as already-ok if the
+    level the compressor is configured with and the clevel byte stored in the repo are the
+    same representation. If they diverge, borg parses and recompresses every object again on
+    every run (see get_csettings).
+    """
+    create_regular_file(archiver.input_path, "file1", size=1024 * 10)
+    create_regular_file(archiver.input_path, "file2", contents=os.urandom(1024 * 10))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input", "-C", "lz4")
+
+    for level in (-1, -4):  # -1 is the level whose byte (255) is also the "no level" sentinel
+        cmd(archiver, "repo-compress", "-C", f"zstd,{level}")
+        repository = Repository(archiver.repository_path, exclusive=True)
+        with repository:
+            manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
+            for id, _ in repo_lister(repository, limit=LIST_SCAN_LIMIT):
+                chunk = repository.get(id, read_data=True)
+                meta, data = manifest.repo_objs.parse(id, chunk, ro_type=ROBJ_DONTCARE)
+                assert meta["ctype"] in (ZSTD.ID, LZ4.ID, CNONE.ID)
+                if meta["ctype"] == ZSTD.ID:
+                    assert meta["clevel"] == ZSTD.encode_level(level)
+
+        # Running it again must recognise the objects as already having the desired
+        # compression. Checking for "0 recompressed" alone would not catch anything: objects
+        # whose settings are misread get recompressed to the identical result and are then
+        # counted as "kept as-is", so the telling number is how many were recognised as ok.
+        output = cmd(archiver, "repo-compress", "-C", f"zstd,{level}", "--stats")
+        stats = re.search(
+            r"Objects: (\d+) total, (\d+) recompressed, (\d+) already had the desired compression", output
+        )
+        assert stats, output
+        total, recompressed, already_ok = (int(g) for g in stats.groups())
+        assert recompressed == 0
+        assert already_ok > 0, f"no object was recognised as already compressed with zstd,{level}"
+
+    cmd(archiver, "check")
+
+
 def test_repo_compress_stats(archiver):
     create_regular_file(archiver.input_path, "file1", size=1024 * 10)
     create_regular_file(archiver.input_path, "file2", contents=os.urandom(1024 * 10))
diff --git a/src/borg/testsuite/compress_test.py b/src/borg/testsuite/compress_test.py
index 35014b8297..77981d94f3 100644
--- a/src/borg/testsuite/compress_test.py
+++ b/src/borg/testsuite/compress_test.py
@@ -232,7 +232,8 @@ def test_default_compression_level(c_type, c_name):
 
 
 @pytest.mark.parametrize(
-    "c_type, c_name, c_levels", [(ZLIB, "zlib", [0, 9]), (LZMA, "lzma", [0, 9]), (ZSTD, "zstd", [1, 22])]
+    "c_type, c_name, c_levels",
+    [(ZLIB, "zlib", [0, 9]), (LZMA, "lzma", [0, 9]), (ZSTD, "zstd", [-128, -22, -4, -1, 1, 22])],
 )
 def test_specified_compression_level(c_type, c_name, c_levels):
     for level in c_levels:
@@ -241,12 +242,53 @@ def test_specified_compression_level(c_type, c_name, c_levels):
         assert cs.level == level
 
 
-@pytest.mark.parametrize("invalid_spec", ["", "lzma,9,invalid", "invalid"])
+@pytest.mark.parametrize("invalid_spec", ["", "lzma,9,invalid", "invalid", "zstd,-129", "zstd,23"])
 def test_invalid_compression_level(invalid_spec):
     with pytest.raises(ArgumentTypeError):
         CompressionSpec(invalid_spec)
 
 
+@pytest.mark.parametrize("level, clevel", [(22, 22), (3, 3), (1, 1), (-1, 255), (-4, 252), (-22, 234), (-128, 128)])
+def test_zstd_level_encoding(level, clevel):
+    """The clevel byte is an int8_t for zstd, and levels 1..22 keep the byte they always had."""
+    assert ZSTD.encode_level(level) == clevel
+    assert ZSTD.decode_level(clevel) == level
+    # the autodetection has to hand back the decoded (possibly negative) level
+    assert Compressor.detect(bytes((ZSTD.ID, clevel))) == (ZSTD, level)
+
+
+@pytest.mark.parametrize("level", [-128, -22, -4, -1, 1, 3, 22])
+def test_zstd_level_roundtrip(level):
+    data = bytes(bytearray(range(256))) * 400
+    compressor = CompressionSpec(f"zstd,{level}").compressor
+    meta, compressed = compressor.compress({}, data)
+    assert meta["clevel"] == ZSTD.encode_level(level)
+    # decompress via the autodetecting Compressor, i.e. the way a repo object is read back
+    meta, decompressed = Compressor("zstd").decompress(dict(meta), compressed)
+    assert decompressed == data
+
+
+@pytest.mark.parametrize("level", [-128, -4, -1, 3])
+def test_zstd_level_roundtrip_legacy_mode(level):
+    data = bytes(bytearray(range(256))) * 400
+    compressor = ZSTD(level=level, legacy_mode=True)
+    # legacy mode returns meta None, but compress() still needs a dict to put "size" into
+    meta, compressed = compressor.compress({}, data)
+    assert compressed[:2] == bytes((ZSTD.ID, ZSTD.encode_level(level)))
+    meta, decompressed = Compressor("zstd", legacy_mode=True).decompress(None, compressed)
+    assert decompressed == data
+
+
+def test_other_compressors_keep_their_level_byte():
+    """Only zstd reinterprets the byte - everything else must store exactly what it did before."""
+    for cls in (CNONE, LZ4, ZLIB, LZMA):
+        assert cls.encode_level(255) == 255
+        assert cls.decode_level(255) == 255
+        for level in (0, 6, 9):
+            assert cls.encode_level(level) == level
+            assert cls.decode_level(level) == level
+
+
 @pytest.mark.parametrize(
     "data_length, expected_padding",
     [

From 17b1c22cd7eec00e88db980ebfae446c0d30d7c5 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Mon, 10 Aug 2026 06:05:28 +0530
Subject: [PATCH 092/144] check: detect missing packs referenced by the index
 (#9898)

With an intact index, load the chunk index and report any referenced
pack id that is absent from packs/. Such packs are counted as errors so
the check fails. Skipped for a corrupt or invalid index, which is rebuilt
from the packs on next use.
---
 src/borg/repository.py                | 36 ++++++++++++++++---
 src/borg/testsuite/repository_test.py | 50 +++++++++++++++++++++++++--
 2 files changed, 80 insertions(+), 6 deletions(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index 120a0f59ec..ae2daf1cfb 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -986,6 +986,11 @@ def check(self, repair=False, max_duration=0, max_age=0):
         it. The record clears at the check that finds the pack intact again or gone (removed by
         compact, or salvaged and dropped by repair; refs #8572); prune() does this from packs/.
 
+        It also reports missing packs (refs #9898): pack ids the chunk index references but that are
+        absent from packs/. The chunk index is loaded and its referenced pack ids are compared with the
+        packs present in the store. A corrupt or invalid index is rebuilt from the packs on next use, so
+        its references are not checked here.
+
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
         max_age, accepting a future timestamp up to MAX_CLOCK_SKEW (clock skew). Results are recorded
         regardless of max_age.
@@ -1027,15 +1032,17 @@ def store_list(namespace):
         t_last_checkpoint = t_start
         index_files = index_errors = 0
         pack_files = pack_errors = pack_skipped = 0
+        missing_pack_ids = []  # packs referenced by the index but absent from packs/ (refs #9898)
         # index and packs get separate progress indicators, each running from 0% to 100%.
         # the index is checked first and in full, on partial checks too: it is small, and index errors
         # stop the pack check below.
         index_infos = store_list("index")
         # an invalid chunk index means an interrupted fragment deletion; it will be rebuilt on next
         # use, so warn rather than verify the leftover fragments.
-        from .cache import chunkindex_is_invalid
+        from .cache import chunkindex_is_invalid, build_chunkindex_from_repo
 
-        if chunkindex_is_invalid(self):
+        index_invalid = chunkindex_is_invalid(self)
+        if index_invalid:
             logger.warning("chunk index is invalid (interrupted operation); it will be rebuilt on next use.")
         index_pi = ProgressIndicatorPercent(total=len(index_infos), msg="Checking index %3.0f%%", msgid="check.index")
         for info in index_infos:
@@ -1104,19 +1111,40 @@ def recorded_ts(info):
                 if pack_infos:
                     pack_pi.show(current=len(pack_infos))  # finish at 100%
                 logger.info("Finished checking packs.")
-            tracker.prune({hex_to_bin(info.name) for info in pack_infos})
+            present_pack_ids = {hex_to_bin(info.name) for info in pack_infos}
+            tracker.prune(present_pack_ids)
             pack_pi.finish()
+            # report packs the index references but that are absent from packs/ (refs #9898). an
+            # invalid index is rebuilt from the packs on next use, so its references are not checked.
+            if not index_invalid:
+                chunks = build_chunkindex_from_repo(self)
+                try:
+                    referenced_pack_ids = {
+                        entry.pack_id
+                        for _, entry in chunks.iteritems()
+                        if not (entry.flags & ChunkIndex.F_PENDING)  # pending: pack_id not resolved yet
+                    }
+                finally:
+                    chunks.clear()
+                missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
-        objs_errors = index_errors + pack_errors
+        objs_errors = index_errors + pack_errors + len(missing_pack_ids)
         summary = (
             f"Checked {index_files} index files ({index_errors} errors) "
             f"and {pack_files} packs ({pack_errors} errors)."
         )
+        if missing_pack_ids:
+            summary += f" {len(missing_pack_ids)} pack(s) referenced by the index are missing."
         if pack_skipped:
             summary += f" Reused {pack_skipped} recent pack check result(s)."
         logger.info(summary)
+        if missing_pack_ids:
+            # one id per line (the list can be long).
+            logger.error(f"Found {len(missing_pack_ids)} missing pack(s) referenced by the index:")
+            for pack_id in missing_pack_ids:
+                logger.error(f"Missing pack: {bin_to_hex(pack_id)}")
         # corrupt_ids() is every pack recorded corrupt, including from earlier runs. with a corrupt
         # index the packs were not scanned, so report nothing.
         corrupt_ids = tracker.corrupt_ids() if index_errors == 0 else []
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 34d39646fb..a42e742045 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1024,6 +1024,15 @@ def boom(*args, **kwargs):
         assert H(1) not in repository._chunks
 
 
+def _serialized_chunkindex(chunks=None):
+    # Serialize a ChunkIndex to bytes, as stored under index/<sha256(content)>. check() parses index
+    # fragments, so a fragment must be a real ChunkIndex serialization.
+    chunks = chunks if chunks is not None else ChunkIndex()
+    with io.BytesIO() as f:
+        chunks.write(f)
+        return f.getvalue()
+
+
 def test_check_detects_corruption_in_later_object(tmp_path):
     # Corruption anywhere in a multi-object pack must be caught, not just in the first object: the pack
     # is named by sha256(content), so flipping any byte makes its stored hash differ from its name.
@@ -1044,7 +1053,7 @@ def test_check_detects_corruption_in_later_object(tmp_path):
 
 def test_check_detects_index_corruption(tmp_path):
     # index/ objects are named by sha256(content) like packs, so check verifies them the same way.
-    content = b"pretend this is a serialized chunk index"
+    content = _serialized_chunkindex()
     index_name = "index/" + bin_to_hex(sha256(content).digest())
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         repository.store_store(index_name, content)
@@ -1094,6 +1103,43 @@ def test_check_intact_multi_object_pack_passes(tmp_path):
         assert repository.check(repair=False) is True
 
 
+def test_check_detects_missing_pack_referenced_by_index(tmp_path, caplog):
+    # check must report a pack the chunk index references but that is absent from packs/ (refs #9898).
+    # put+flush+close persists the index; then delete the pack, keeping its index entry.
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()  # flush before close persists the index
+    with Repository(location, exclusive=True) as repository:
+        assert repository.check(repair=False) is True  # index and pack both present
+        pack_id = repository.chunks[H(0)].pack_id
+        repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone, index entry kept
+        with caplog.at_level(logging.ERROR):
+            assert repository.check(repair=False) is False
+        assert f"Missing pack: {bin_to_hex(pack_id)}" in caplog.text
+
+
+def test_check_missing_pack_detection_skipped_when_index_invalid(tmp_path, caplog):
+    # an invalid index is rebuilt from the packs on next use, so check does not report missing packs
+    # from it (refs #9898); it only warns about the invalid index.
+    from ..cache import write_chunkindex_invalid
+
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()
+    with Repository(location, exclusive=True) as repository:
+        pack_id = repository.chunks[H(0)].pack_id
+        repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone
+        write_chunkindex_invalid(repository)  # mark the index invalid
+        with caplog.at_level(logging.WARNING):
+            assert repository.check(repair=False) is True
+        assert "Missing pack" not in caplog.text
+        assert "chunk index is invalid" in caplog.text
+
+
 def test_check_checked_packs_roundtrip(tmp_path):
     # the set survives a store/load round-trip; a rotted blob loads as empty.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
@@ -1574,7 +1620,7 @@ def finish(self, *args, **kwargs):
     monkeypatch.setattr("borg.repository.ProgressIndicatorPercent", FakePI)
     pack = fchunk(b"A", chunk_id=H(1))
     pack_name = "packs/" + bin_to_hex(sha256(pack).digest())
-    index_content = b"serialized chunk index"
+    index_content = _serialized_chunkindex()
     index_name = "index/" + bin_to_hex(sha256(index_content).digest())
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         repository.store_store(pack_name, pack)

From dfcc12b80d1eb0caaa39d68e94cbaa16d7a2f9a0 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 12 Aug 2026 14:53:38 +0530
Subject: [PATCH 093/144] check: report orphan packs, tidy missing-pack
 messages (#9898)

Report packs present in packs/ that no index entry references at info level;
they are interrupted-operation leftovers, not errors. Reword the missing-pack
error to match the summary and correct the invalid-index comment.
---
 src/borg/repository.py                | 19 ++++++++++++++-----
 src/borg/testsuite/repository_test.py | 17 +++++++++++++++++
 2 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index ae2daf1cfb..35b4c50baa 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1037,8 +1037,8 @@ def store_list(namespace):
         # the index is checked first and in full, on partial checks too: it is small, and index errors
         # stop the pack check below.
         index_infos = store_list("index")
-        # an invalid chunk index means an interrupted fragment deletion; it will be rebuilt on next
-        # use, so warn rather than verify the leftover fragments.
+        # an interrupted fragment deletion leaves the invalid marker set; the index is rebuilt on next
+        # use, so warn rather than fail.
         from .cache import chunkindex_is_invalid, build_chunkindex_from_repo
 
         index_invalid = chunkindex_is_invalid(self)
@@ -1114,8 +1114,9 @@ def recorded_ts(info):
             present_pack_ids = {hex_to_bin(info.name) for info in pack_infos}
             tracker.prune(present_pack_ids)
             pack_pi.finish()
-            # report packs the index references but that are absent from packs/ (refs #9898). an
-            # invalid index is rebuilt from the packs on next use, so its references are not checked.
+            # cross-check the chunk index against packs/ (refs #9898). skipped for an invalid index:
+            # its fragment set may be incomplete, and build_chunkindex_from_repo would delete fragments
+            # and rebuild (a write), which a read-only check must not do.
             if not index_invalid:
                 chunks = build_chunkindex_from_repo(self)
                 try:
@@ -1126,7 +1127,15 @@ def recorded_ts(info):
                     }
                 finally:
                     chunks.clear()
+                # index entries pointing to a pack absent from packs/: data loss.
                 missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
+                # packs no index entry references: leftovers of an interrupted operation, not an error.
+                orphan_pack_ids = sorted(present_pack_ids - referenced_pack_ids)
+                if orphan_pack_ids:
+                    logger.info(
+                        f"{len(orphan_pack_ids)} pack(s) in packs/ are not referenced by the index "
+                        f"(leftovers from an interrupted operation)."
+                    )
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
@@ -1142,7 +1151,7 @@ def recorded_ts(info):
         logger.info(summary)
         if missing_pack_ids:
             # one id per line (the list can be long).
-            logger.error(f"Found {len(missing_pack_ids)} missing pack(s) referenced by the index:")
+            logger.error(f"{len(missing_pack_ids)} pack(s) referenced by the index are missing:")
             for pack_id in missing_pack_ids:
                 logger.error(f"Missing pack: {bin_to_hex(pack_id)}")
         # corrupt_ids() is every pack recorded corrupt, including from earlier runs. with a corrupt
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index a42e742045..8c2c6d411e 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1140,6 +1140,23 @@ def test_check_missing_pack_detection_skipped_when_index_invalid(tmp_path, caplo
         assert "chunk index is invalid" in caplog.text
 
 
+def test_check_reports_orphan_pack_not_referenced_by_index(tmp_path, caplog):
+    # a pack that no index entry references is reported at info level and does not fail check (refs #9898).
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()  # flush before close persists the index
+    with Repository(location, exclusive=True) as repository:
+        # a validly-named pack (name == sha256(content)) that no index entry points into.
+        content = b"orphan pack content"
+        orphan_id = sha256(content).digest()
+        repository.store_store("packs/" + bin_to_hex(orphan_id), content)
+        with caplog.at_level(logging.INFO):
+            assert repository.check(repair=False) is True
+        assert "not referenced by the index" in caplog.text
+
+
 def test_check_checked_packs_roundtrip(tmp_path):
     # the set survives a store/load round-trip; a rotted blob loads as empty.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:

From 3c07f9e204de4bf4adb9d1c6a5c11780ed92281c Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 13:12:04 +0200
Subject: [PATCH 094/144] update CHANGES

---
 docs/changes.rst | 58 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 319ae5410f..f51e34bb67 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -177,11 +177,10 @@ New features:
   - give each thread its own LZ4 scratch buffer, #10032
 - faster extract / mount:
 
-  - avoid refetching/reparsing repeated chunks, #1678
+  - avoid refetching/reparsing repeated chunks: serve all-zero chunks without
+    repository access, cache recently parsed chunks, #1678
   - do not verify the chunk id on every read from an encrypted repo (the
     AEAD authentication covers reads), see BORG_ASSERT_ID, #9994, #7362
-  - serve all-zero chunks without repository access; also cache
-    recently parsed chunks, #1678
 - more, faster, and more secure chunkers:
 
   - fastcdc is the new and faster default chunker, #9957
@@ -189,22 +188,29 @@ New features:
 
     - AVX-512 / AVX2 on x86-64 (Intel / AMD), NEON on aarch64, plus a portable
       blockwise one; all bit-identical to the sequential loop
-    - the sequential loop is what runs unless BORG_FASTCDC_KERNEL /
-      BORG_BUZHASH64_KERNEL / BORG_AES_CHUNKER_KERNEL selects another kernel:
+    - the sequential loop is what runs unless one of BORG_FASTCDC_KERNEL /
+      BORG_BUZHASH64_KERNEL / BORG_AES_CHUNKER_KERNEL selects another kernel;
       which one is fastest depends on the CPU *and* the compiler, so nothing
       is chosen automatically
   - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers
-    (UHF-then-PRF), with direct AES hw acceleration (AES-NI or VAES/AVX-512) or
-    via OpenSSL, #9987, #10043
+    (UHF-then-PRF), with direct AES hardware acceleration (AES-NI or
+    VAES/AVX-512) or via OpenSSL, #9987, #10043
   - zero-copy fill and lazy buffer compaction optimizations
   - log the chunker and its scan kernel at debug level
 - compression: support zstd's negative ("fast") levels, ``zstd,-1`` .. ``zstd,-128``, #9950.
   They trade compression ratio for speed. Compatible with existing repositories.
+- check:
+
+  - keep pack check results, add --max-age to reuse them, #9696, #9925
+  - report missing chunks grouped as chunk -> files -> archives, #9218, #9965
+  - calendar-aware --max-age, symmetric clock-skew window
+  - stream one line per missing chunk id, run report on abort, lower report caps, #9218
+- help environment: new help topic about environment variables, #10061
 - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice
   replacement for `borg mount` in some use cases, #9942
 - mount: expose POSIX ACLs on Linux mounts (not enforced), #1042
 - analyze: report deduplicated size of a set of archives, #5741
-- repo-compress: was temporarily gone, re-added now with pack support, #9663
+- repo-compress: was temporarily gone, now re-added with pack support, #9663
 - version: add --json output, #10004
 - benchmark cpu:
 
@@ -218,13 +224,6 @@ New features:
 - completion: generate fish and tcsh completions, #9989, #9503
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
-- check:
-
-  - keep pack check results, add --max-age to reuse them, #9696, #9925
-  - report missing chunks grouped as chunk -> files -> archives, #9218, #9965
-  - calendar-aware --max-age, symmetric clock-skew window
-  - check: stream one line per missing chunk id, run report on abort, lower report caps, #9218
-
 
 Fixes:
 
@@ -237,7 +236,13 @@ Fixes:
 - release chunk data memoryviews (fixes PyPy memory leak), #1755, #9978
 - fix DownloadPipeline.fetch_many() crashing on a missing chunk, #10024
 - lrucache: make it thread-safe
-- crypto: start a new session after encrypting 2TiB with one aes256-ocb session key, #6501
+- crypto: start a new session after encrypting 2 TiB with one aes256-ocb session key, #6501
+- check:
+
+  - flush pack writer in ArchiveChecker.finish() before dropping the index
+  - handle Ctrl-C at safe boundaries, #7893, #9966
+  - report invalid pack names instead of crashing
+  - honest per-run interrupt count, drop redundant save, fix stale SIGINT docs
 
 Other changes:
 
@@ -248,6 +253,14 @@ Other changes:
 - crypto: raise IntegrityError for truncated AEAD/AE envelopes
 - chunkers: refactor the shared machinery into a common ChunkerBase class
 - PackReader.read(): return a memoryview of the in-memory pack instead of copying
+- remove avoidable per-chunk memory copies on the hot data path, #10059, #10060
+
+  - compress: lz4 decompresses directly into the result bytes object
+  - compress: do not copy chunk data to bytes, use the buffer protocol
+  - chunkers: read file data directly into the caller's buffer (if possible)
+  - crypto: AEAD encrypt/decrypt directly into the result bytes object
+- write_chunkindex_to_repo: reduce memory needs, #9886
+- export-tar/import-tar: zstd (de)compression is in-process now, #10067
 - mount/webdav: unify the 3 archive-as-filesystem implementations, #10020.
   Behavior changes that fell out of the unification:
 
@@ -279,6 +292,19 @@ Other changes:
   - FAME.md: update contributor statistics, #10022
   - crypto: misc. improvements to code and docs, #6501, ...
   - GitHub issue #10000: "We Are Borg" joke collection
+- CI / tests:
+
+  - give the test VMs 4 CPUs / 8 GiB RAM
+  - cache pip-built wheels (Windows, BSDs, OmniOS, Haiku)
+  - upgrade cross-platform-actions to 1.4.0
+  - upgrade to NetBSD 11.0
+  - upgrade to OpenBSD 7.9
+  - OpenBSD: put TMPDIR on an mfs
+  - fix VM job hangs, use all runner CPUs
+  - fail hung tests after 5 minutes on the test VMs
+  - time out the "Start VM" step after 15 minutes
+  - time-bound LRUCache.test_threaded_access
+  - benchmark crud json-lines: I/O throughput may round to 0
 
 
 Version 2.0.0b22 (2026-07-22)

From 98b8b4f7a326ea64aa81ce184f24c9551858d4f7 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 15:28:19 +0200
Subject: [PATCH 095/144] create --progress: show big sizes with more decimals,
 fixes #3559

At TB scale, the sizes shown by "borg create --progress" only changed
every 10GB, so one could not tell whether it was still making progress.

format_file_size (and the sizeof_fmt* helpers) now have a "fine" option
giving big sizes a scale dependent precision, so that changes of ~1MB
stay visible: GB: 3, TB: 6, PB: 9 decimals (capped at 9, a float can not
resolve more anyway). Unit selection is not affected, so a size is always
shown with the same unit as before.

More decimals eat into the space left for the path, so on a terminal they
are only used if it is at least 110 columns wide - even in the worst case,
that still leaves more room for the path than 80 columns did before. If
the output does not go to a terminal (e.g. into a logfile), the precise
format is always used. The JSON progress output is unchanged.
---
 docs/usage/create.rst.inc                     |  5 ++
 src/borg/archive.py                           | 12 ++++-
 src/borg/archiver/create_cmd.py               |  5 ++
 src/borg/helpers/parseformat.py               | 27 +++++++---
 src/borg/testsuite/archive_test.py            | 33 ++++++++++--
 .../testsuite/helpers/parseformat_test.py     | 51 +++++++++++++++++++
 6 files changed, 119 insertions(+), 14 deletions(-)

diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc
index 529eccb66e..d53f695388 100644
--- a/docs/usage/create.rst.inc
+++ b/docs/usage/create.rst.inc
@@ -263,6 +263,11 @@ The ``--progress`` option shows (from left to right) Original and (uncompressed)
 deduplicated size (O and U respectively), then the Number of files (N) processed so far,
 followed by the currently processed path.
 
+Sizes of GB and above are shown with enough decimal places that even MB-sized progress
+stays visible. On a terminal, this needs a width of at least 110 columns - on narrower
+terminals, the compact format is used, so that the path stays readable. If the output
+does not go to a terminal (e.g. into a logfile), the precise format is always used.
+
 When using ``--stats``, you will get some statistics about how much data was
 added - the "This Archive" deduplicated size there is most interesting as that is
 how much your repository will grow. Please note that the "All archives" stats refer to
diff --git a/src/borg/archive.py b/src/borg/archive.py
index 6a9d10623f..196d20f2e5 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -213,7 +213,10 @@ def show_progress(self, item=None, final=False, stream=None):
             elif not stream.isatty():
                 # Non-TTY output: use normal linefeeds and do not truncate the path.
                 if not final:
-                    msg = "{0.osize_fmt} O {0.usize_fmt} U {0.nfiles} N ".format(self)
+                    # no width limit here, so always show the sizes precisely, see #3559.
+                    osize_fmt = format_file_size(self.osize, fine=True)
+                    usize_fmt = format_file_size(self.usize, fine=True)
+                    msg = f"{osize_fmt} O {usize_fmt} U {self.nfiles} N "
                     msg += remove_surrogates(item.path) if item else ""
                 else:
                     msg = ""
@@ -221,7 +224,12 @@ def show_progress(self, item=None, final=False, stream=None):
             else:
                 columns, lines = get_terminal_size()
                 if not final:
-                    msg = "{0.osize_fmt} O {0.usize_fmt} U {0.nfiles} N ".format(self)
+                    # more decimals eat into the space left for the path, so only show them
+                    # if the terminal is clearly wider than the classic 80 columns, see #3559.
+                    fine = columns >= 110
+                    osize_fmt = format_file_size(self.osize, fine=fine)
+                    usize_fmt = format_file_size(self.usize, fine=fine)
+                    msg = f"{osize_fmt} O {usize_fmt} U {self.nfiles} N "
                     path = remove_surrogates(item.path) if item else ""
                     space = columns - swidth(msg)
                     if space < 12:
diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py
index 7eeb70642b..87f183880a 100644
--- a/src/borg/archiver/create_cmd.py
+++ b/src/borg/archiver/create_cmd.py
@@ -656,6 +656,11 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         deduplicated size (O and U respectively), then the Number of files (N) processed so far,
         followed by the currently processed path.
 
+        Sizes of GB and above are shown with enough decimal places that even MB-sized progress
+        stays visible. On a terminal, this needs a width of at least 110 columns - on narrower
+        terminals, the compact format is used, so that the path stays readable. If the output
+        does not go to a terminal (e.g. into a logfile), the precise format is always used.
+
         When using ``--stats``, you will get some statistics about how much data was
         added - the "This Archive" deduplicated size there is most interesting as that is
         how much your repository will grow. Please note that the "All archives" stats refer to
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index 1ff34fb32a..7c6e920d86 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -554,15 +554,18 @@ def get_size_units():
     return "si"
 
 
-def format_file_size(v, precision=2, sign=False):
-    """Format file size into a human friendly format, using the units requested via BORG_UNITS."""
+def format_file_size(v, precision=2, sign=False, fine=False):
+    """Format file size into a human friendly format, using the units requested via BORG_UNITS.
+
+    If fine is given, big sizes get more decimals, so that changes of ~1MB stay visible.
+    """
     units = get_size_units()
     if units == "raw":
-        # exact byte counts, so scripts can easily parse the output
+        # exact byte counts, so scripts can easily parse the output (fine does not apply)
         v = round(v)  # v might be a float, e.g. a throughput value
         return f"{v:{'+' if sign and v > 0 else ''}d} B"
     fn = sizeof_fmt_iec if units == "iec" else sizeof_fmt_decimal
-    return fn(v, suffix="B", sep=" ", precision=precision, sign=sign)
+    return fn(v, suffix="B", sep=" ", precision=precision, sign=sign, fine=fine)
 
 
 class FileSize(int):
@@ -587,39 +590,47 @@ def parse_file_size(s):
     return int(float(s) * factor)
 
 
-def sizeof_fmt(num, suffix="B", units=None, power=None, sep="", precision=2, sign=False):
+def sizeof_fmt(num, suffix="B", units=None, power=None, sep="", precision=2, sign=False, fine=False):
     sign = "+" if sign and num > 0 else ""
     fmt = "{0:{1}.{2}f}{3}{4}{5}"
     prec = 0
-    for unit in units[:-1]:
+    for i, unit in enumerate(units[:-1]):
         if abs(round(num, precision)) < power:
             break
         num /= float(power)
         prec = precision
     else:
+        i = len(units) - 1
         unit = units[-1]
+    if fine:
+        # scale dependent precision, so ~1 M<unit> stays visible at any scale: M: 2 (default),
+        # G: 3, T: 6, P: 9 decimals. capped at 9 - a float can not resolve more anyway and it
+        # bounds the output width. see #3559.
+        prec = max(prec, min((i - 2) * 3, 9))
     return fmt.format(num, sign, prec, sep, unit, suffix)
 
 
-def sizeof_fmt_iec(num, suffix="B", sep="", precision=2, sign=False):
+def sizeof_fmt_iec(num, suffix="B", sep="", precision=2, sign=False, fine=False):
     return sizeof_fmt(
         num,
         suffix=suffix,
         sep=sep,
         precision=precision,
         sign=sign,
+        fine=fine,
         units=["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"],
         power=1024,
     )
 
 
-def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False):
+def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False, fine=False):
     return sizeof_fmt(
         num,
         suffix=suffix,
         sep=sep,
         precision=precision,
         sign=sign,
+        fine=fine,
         units=["", "k", "M", "G", "T", "P", "E", "Z", "Y"],
         power=1000,
     )
diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py
index 81214ba468..8da624c9da 100644
--- a/src/borg/testsuite/archive_test.py
+++ b/src/borg/testsuite/archive_test.py
@@ -35,6 +35,11 @@ def show_progress_force(stats, **kwargs):
     stats.show_progress(**kwargs)
 
 
+class TTYStringIO(StringIO):
+    def isatty(self):
+        return True
+
+
 def test_stats_basic(stats):
     assert stats.osize == 20
     assert stats.usize == 20
@@ -44,10 +49,6 @@ def test_stats_basic(stats):
 
 
 def test_stats_progress_tty(stats, monkeypatch, columns=80):
-    class TTYStringIO(StringIO):
-        def isatty(self):
-            return True
-
     monkeypatch.setenv("COLUMNS", str(columns))
     out = TTYStringIO()
     show_progress_force(stats, stream=out)
@@ -69,6 +70,23 @@ def isatty(self):
     assert out.getvalue() == s + buf + "\r"
 
 
+@pytest.mark.parametrize(
+    "columns, s",
+    [
+        (80, "1.00 TB O 20 B U 1 N foo"),  # narrow terminal: compact format, so the path stays visible
+        (109, "1.00 TB O 20 B U 1 N foo"),  # still too narrow
+        (110, "1.000000 TB O 20 B U 1 N foo"),  # wide terminal: more decimals, see #3559
+    ],
+)
+def test_stats_progress_tty_fine(stats, monkeypatch, columns, s):
+    monkeypatch.setenv("COLUMNS", str(columns))
+    stats.update(10**12, unique=False)
+    out = TTYStringIO()
+    show_progress_force(stats, item=Item(path="foo"), final=False, stream=out)
+    buf = " " * (columns - len(s))
+    assert out.getvalue() == s + buf + "\r"
+
+
 def test_stats_progress_file(stats, monkeypatch):
     out = StringIO()
     show_progress_force(stats, stream=out)
@@ -88,6 +106,13 @@ def test_stats_progress_file(stats, monkeypatch):
     s = f"1.02 kB O 20 B U 1 N {path}"
     assert out.getvalue() == s + "\n"
 
+    out = StringIO()
+    stats.update(10**12, unique=False)
+    path = "foo"
+    show_progress_force(stats, item=Item(path=path), final=False, stream=out)
+    s = f"1.000000 TB O 20 B U 1 N {path}"  # no width limit here, so always more decimals, see #3559
+    assert out.getvalue() == s + "\n"
+
 
 def test_stats_progress_rate_limited(stats):
     out = StringIO()
diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py
index 6399b1165d..9c47d8153e 100644
--- a/src/borg/testsuite/helpers/parseformat_test.py
+++ b/src/borg/testsuite/helpers/parseformat_test.py
@@ -678,6 +678,57 @@ def test_file_size_sign(size, fmt):
     assert format_file_size(size, sign=True) == fmt
 
 
+@pytest.mark.parametrize(
+    "size, fmt",
+    [
+        (0, "0 B"),  # bytes and kB are formatted like with fine=False
+        (999, "999 B"),
+        (1234, "1.23 kB"),
+        (10**6, "1.00 MB"),  # MB: 2 decimals, like with fine=False
+        (1234567, "1.23 MB"),
+        (10**9, "1.000 GB"),  # GB: 3 decimals
+        (1234567890, "1.235 GB"),
+        (999995000, "1.000 GB"),  # rounded up to the next unit, like with fine=False
+        (999994999999, "999.995 GB"),  # almost 1 TB, but still shown as GB
+        (999999000000, "0.999999 TB"),  # unit like with fine=False ("1.00 TB"), but exact value
+        (10**12, "1.000000 TB"),  # TB: 6 decimals
+        (5000001000000, "5.000001 TB"),  # 1 MB more is visible now, that is the point of #3559
+        (10**15, "1.000000000 PB"),  # PB: 9 decimals
+        (5 * 10**15 + 10**6, "5.000000001 PB"),
+        (10**18, "1.000000000 EB"),  # decimals are capped at 9
+        (-(10**12), "-1.000000 TB"),
+    ],
+)
+def test_file_size_fine(size, fmt):
+    """fine=True shows big sizes with more decimals, so that changes of ~1MB stay visible"""
+    assert format_file_size(size, fine=True) == fmt
+
+
+@pytest.mark.parametrize(
+    "size, fmt",
+    [
+        (2**20, "1.00 MiB"),
+        (2**30, "1.000 GiB"),
+        (2**40, "1.000000 TiB"),
+        (2**40 + 2**20, "1.000001 TiB"),
+        (2**50 + 2**20, "1.000000001 PiB"),
+    ],
+)
+def test_file_size_fine_iec(monkeypatch, size, fmt):
+    monkeypatch.setenv("BORG_UNITS", "iec")
+    assert format_file_size(size, fine=True) == fmt
+
+
+def test_file_size_fine_raw(monkeypatch):
+    """BORG_UNITS=raw is exact anyway, so fine does not change anything"""
+    monkeypatch.setenv("BORG_UNITS", "raw")
+    assert format_file_size(10**12, fine=True) == "1000000000000 B"
+
+
+def test_file_size_fine_sign():
+    assert format_file_size(10**12, sign=True, fine=True) == "+1.000000 TB"
+
+
 @pytest.mark.parametrize(
     "string, value", [("1", 1), ("20", 20), ("5K", 5000), ("1.75M", 1750000), ("1e+9", 1e9), ("-1T", -1e12)]
 )

From 5212ded8304b6cff32b1f8726af9be85e179fbef Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 15:59:01 +0200
Subject: [PATCH 096/144] helpers: add sorting.py for --sort-by style
 multi-field sorting

Spec parsing/validation and the stable, direction-aware multi-field sort of
"--sort-by SPEC" options, so that commands only have to provide the sort key
extraction for their own fields.
---
 src/borg/helpers/sorting.py                | 68 ++++++++++++++++++++
 src/borg/testsuite/helpers/sorting_test.py | 74 ++++++++++++++++++++++
 2 files changed, 142 insertions(+)
 create mode 100644 src/borg/helpers/sorting.py
 create mode 100644 src/borg/testsuite/helpers/sorting_test.py

diff --git a/src/borg/helpers/sorting.py b/src/borg/helpers/sorting.py
new file mode 100644
index 0000000000..e1091e354a
--- /dev/null
+++ b/src/borg/helpers/sorting.py
@@ -0,0 +1,68 @@
+"""
+Multi-field, direction-aware, stable sorting of command output.
+
+Used by commands offering a ``--sort-by SPEC`` option that sorts archive contents
+(``borg list``, ``borg diff``), where SPEC is a comma-separated list of field names,
+each optionally prefixed with "<" (ascending, the default) or ">" (descending).
+
+Not to be confused with helpers.parseformat.SortBySpec, which is the simpler
+archive-level sort spec of ``borg repo-list`` (no direction prefixes).
+"""
+
+from .argparsing import ArgumentTypeError
+
+ASCENDING, DESCENDING = "<", ">"
+
+
+def parse_sort_spec(spec):
+    """Split SPEC into [(field, descending), ...]. None / empty / blank gives []."""
+    result = []
+    for part in (spec or "").split(","):
+        part = part.strip()
+        if not part:
+            continue
+        if part[0] in (ASCENDING, DESCENDING):
+            result.append((part[1:], part[0] == DESCENDING))
+        else:
+            result.append((part, False))
+    return result
+
+
+def sort_spec_validator(allowed_keys, *, name="sort_spec"):
+    """Build an argparse type= callable validating a --sort-by SPEC.
+
+    The returned function has the allowed keys as .sort_keys, so shell completion
+    can be generated from it, see archiver/completion_cmd.py.
+    """
+    allowed = tuple(allowed_keys)
+
+    def validator(text):
+        parsed = parse_sort_spec(text)
+        if not parsed:
+            raise ArgumentTypeError("unsupported sort field: empty spec")
+        for field, _ in parsed:
+            if field not in allowed:
+                raise ArgumentTypeError(f"unsupported sort field: {field}, supported: {', '.join(allowed)}")
+        return ",".join((DESCENDING if descending else "") + field for field, descending in parsed)
+
+    validator.__name__ = name
+    validator.sort_keys = allowed
+    return validator
+
+
+def sorted_by_spec(seq, spec, key_for):
+    """Sort seq as requested by spec, using key_for(field, element) to compute sort keys.
+
+    Sorting is stable and applied from the last field of the spec to the first, so the
+    first field is the primary sort criterion.
+
+    If spec does not select any field, seq is returned as is (and not consumed), so
+    callers can keep streaming.
+    """
+    parsed = parse_sort_spec(spec)
+    if not parsed:
+        return seq
+    result = list(seq)
+    for field, descending in reversed(parsed):
+        result.sort(key=lambda element, field=field: key_for(field, element), reverse=descending)
+    return result
diff --git a/src/borg/testsuite/helpers/sorting_test.py b/src/borg/testsuite/helpers/sorting_test.py
new file mode 100644
index 0000000000..9901e51df5
--- /dev/null
+++ b/src/borg/testsuite/helpers/sorting_test.py
@@ -0,0 +1,74 @@
+import pytest
+
+from ...helpers.argparsing import ArgumentTypeError
+from ...helpers.sorting import parse_sort_spec, sort_spec_validator, sorted_by_spec
+
+
+@pytest.mark.parametrize(
+    "spec, expected",
+    [
+        (None, []),
+        ("", []),
+        ("   ", []),
+        (",", []),
+        ("path", [("path", False)]),
+        ("<path", [("path", False)]),
+        (">path", [("path", True)]),
+        (" >size , path ", [("size", True), ("path", False)]),
+        ("size,,path", [("size", False), ("path", False)]),
+    ],
+)
+def test_parse_sort_spec(spec, expected):
+    assert parse_sort_spec(spec) == expected
+
+
+def test_validator_accepts_and_normalizes():
+    validator = sort_spec_validator(("path", "size"))
+    assert validator("path") == "path"
+    assert validator("<path") == "path"  # "<" is the default and thus dropped
+    assert validator(" >size , path ") == ">size,path"
+
+
+def test_validator_rejects():
+    validator = sort_spec_validator(("path", "size"))
+    for spec in ("", "  ", ",", "nosuchfield", "path,nosuchfield", ">", "<"):
+        with pytest.raises(ArgumentTypeError):
+            validator(spec)
+
+
+def test_validator_has_sort_keys():
+    validator = sort_spec_validator(["path", "size"], name="test_sort_spec")
+    assert validator.sort_keys == ("path", "size")
+    assert validator.__name__ == "test_sort_spec"
+
+
+def key_for(field, element):
+    return element[field]
+
+
+def test_sorted_by_spec_keeps_unsorted_seq_as_is():
+    # important: without a sort spec, callers shall be able to keep streaming
+    seq = iter([{"path": "b"}, {"path": "a"}])
+    assert sorted_by_spec(seq, None, key_for) is seq
+    assert sorted_by_spec(seq, "", key_for) is seq
+
+
+def test_sorted_by_spec_single_field():
+    seq = [{"path": "b"}, {"path": "c"}, {"path": "a"}]
+    assert [e["path"] for e in sorted_by_spec(seq, "path", key_for)] == ["a", "b", "c"]
+    assert [e["path"] for e in sorted_by_spec(seq, ">path", key_for)] == ["c", "b", "a"]
+
+
+def test_sorted_by_spec_multiple_fields():
+    seq = [{"size": 1, "path": "b"}, {"size": 2, "path": "c"}, {"size": 1, "path": "a"}, {"size": 2, "path": "a"}]
+    # first field is the primary criterion, following fields break ties
+    result = [(e["size"], e["path"]) for e in sorted_by_spec(seq, ">size,path", key_for)]
+    assert result == [(2, "a"), (2, "c"), (1, "a"), (1, "b")]
+    result = [(e["size"], e["path"]) for e in sorted_by_spec(seq, "size,>path", key_for)]
+    assert result == [(1, "b"), (1, "a"), (2, "c"), (2, "a")]
+
+
+def test_sorted_by_spec_is_stable():
+    # equal sort keys must not change the relative order of the elements
+    seq = [{"size": 1, "path": "b"}, {"size": 1, "path": "a"}, {"size": 1, "path": "c"}]
+    assert [e["path"] for e in sorted_by_spec(seq, "size", key_for)] == ["b", "a", "c"]

From cc44942e78d24deb8b19d013df778dbd43a5c8d0 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 15:59:12 +0200
Subject: [PATCH 097/144] diff: use the shared sorting helper, no behaviour
 change

The sort spec parsing/validation and the sorting itself now come from
helpers.sorting, only the sort key extraction stays here. The validator is a
module level object now, so shell completion can be attached to it.
---
 src/borg/archiver/diff_cmd.py | 72 +++++++++++------------------------
 1 file changed, 23 insertions(+), 49 deletions(-)

diff --git a/src/borg/archiver/diff_cmd.py b/src/borg/archiver/diff_cmd.py
index f7d82a5939..30e378bf28 100644
--- a/src/borg/archiver/diff_cmd.py
+++ b/src/borg/archiver/diff_cmd.py
@@ -8,13 +8,32 @@
 from ..constants import *  # NOQA
 from ..helpers import BaseFormatter, DiffFormatter, archivename_validator, PathSpec, BorgJsonEncoder
 from ..helpers import IncludePatternNeverMatchedWarning, remove_surrogates
-from ..helpers.argparsing import ArgumentParser, ArgumentTypeError
+from ..helpers.argparsing import ArgumentParser
+from ..helpers.sorting import sort_spec_validator, sorted_by_spec
 from ..item import ItemDiff
 from ..manifest import Manifest
 from ..logger import create_logger
 
 logger = create_logger()
 
+# item diff fields "borg diff --sort-by" can sort by
+DIFF_SORT_KEYS = (
+    "path",
+    "size_added",
+    "size_removed",
+    "size_diff",
+    "size",
+    "user",
+    "group",
+    "uid",
+    "gid",
+    "ctime",
+    "mtime",
+    "ctime_diff",
+    "mtime_diff",
+)
+diff_sort_spec = sort_spec_validator(DIFF_SORT_KEYS, name="diff_sort_spec")
+
 
 class DiffMixIn:
     @with_repository(compatibility=(Manifest.Operation.READ,))
@@ -93,19 +112,9 @@ def print_text_output(diff, formatter):
         # Filter out equal items early (keep as generator; listify only if sorting)
         diffs = (diff for diff in diffs_iter if not diff.equal(args.content_only))
 
-        sort_specs = []
-        if args.sort_by:
-            for spec in args.sort_by.split(","):
-                spec = spec.strip()
-                if spec:
-                    sort_specs.append(spec)
-
         def key_for(field: str, d: "ItemDiff"):
-            # strip direction markers if present
-            if field and field[0] in ("<", ">"):
-                field = field[1:]
             # path
-            if field in (None, "", "path"):
+            if field == "path":
                 return remove_surrogates(d.path)
             # compute size_* from changes
             if field in ("size_diff", "size_added", "size_removed"):
@@ -150,15 +159,7 @@ def key_for(field: str, d: "ItemDiff"):
                 return it.get(field, attr_defaults[field])
             raise ValueError(f"Invalid field name: {field}")
 
-        if sort_specs:
-            diffs = list(diffs)
-            # Apply stable sorts from last to first
-            for spec in reversed(sort_specs):
-                desc = False
-                field = spec
-                if field and field[0] in ("<", ">"):
-                    desc = field[0] == ">"
-                diffs.sort(key=lambda di: key_for(field, di), reverse=desc)
+        diffs = sorted_by_spec(diffs, args.sort_by, key_for)
 
         formatter = DiffFormatter(format, args.content_only)
         for diff in diffs:
@@ -266,33 +267,6 @@ def build_parser_diff(self, subparsers, common_parser, mid_common_parser):
             )
         )
 
-        def diff_sort_spec_validator(s):
-            if not isinstance(s, str):
-                raise ArgumentTypeError("unsupported sort field (not a string)")
-            allowed = {
-                "path",
-                "size_added",
-                "size_removed",
-                "size_diff",
-                "size",
-                "user",
-                "group",
-                "uid",
-                "gid",
-                "ctime",
-                "mtime",
-                "ctime_diff",
-                "mtime_diff",
-            }
-            parts = [p.strip() for p in s.split(",") if p.strip()]
-            if not parts:
-                raise ArgumentTypeError("unsupported sort field: empty spec")
-            for spec in parts:
-                field = spec[1:] if spec and spec[0] in (">", "<") else spec
-                if field not in allowed:
-                    raise ArgumentTypeError(f"unsupported sort field: {field}")
-            return ",".join(parts)
-
         subparser = ArgumentParser(parents=[common_parser], description=self.do_diff.__doc__, epilog=diff_epilog)
         subparsers.add_subcommand("diff", subparser, help="find differences in archive contents")
         subparser.add_argument(
@@ -318,7 +292,7 @@ def diff_sort_spec_validator(s):
         subparser.add_argument(
             "--sort-by",
             dest="sort_by",
-            type=diff_sort_spec_validator,
+            type=diff_sort_spec,
             help="Sort output by comma-separated fields (e.g., '>size_added,path').",
         )
         subparser.add_argument(

From 43b29eea9ff53ce026d03aefb292dd8d88841797 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 15:59:25 +0200
Subject: [PATCH 098/144] list: add --sort-by=SPEC, fixes #9009

SPEC is a comma-separated list of fields, each optionally prefixed with ">"
for descending or "<" for ascending (the default) order. The sorts are
applied stably from the last field to the first, thus the first field is the
primary sort criterion, e.g.: borg list --sort-by='>size,path' ARCHIVE

Sorting needs all (matching) items in memory, so without --sort-by, the items
are still streamed and their order stays undefined.
---
 src/borg/archiver/list_cmd.py | 71 ++++++++++++++++++++++++++++++++++-
 1 file changed, 70 insertions(+), 1 deletion(-)

diff --git a/src/borg/archiver/list_cmd.py b/src/borg/archiver/list_cmd.py
index ffbf9c48c5..1893b0df44 100644
--- a/src/borg/archiver/list_cmd.py
+++ b/src/borg/archiver/list_cmd.py
@@ -1,4 +1,5 @@
 import os
+import stat
 import textwrap
 import sys
 
@@ -8,12 +9,39 @@
 from ..constants import *  # NOQA
 from ..helpers import ItemFormatter, BaseFormatter, archivename_validator, PathSpec
 from ..helpers.argparsing import ArgumentParser
+from ..helpers.sorting import sort_spec_validator, sorted_by_spec
 from ..manifest import Manifest
 
 from ..logger import create_logger
 
 logger = create_logger()
 
+# item fields "borg list --sort-by" can sort by
+ITEM_SORT_KEYS = ("path", "type", "mode", "user", "uid", "group", "gid", "size", "mtime", "ctime", "atime")
+item_sort_spec = sort_spec_validator(ITEM_SORT_KEYS, name="item_sort_spec")
+
+
+def item_sort_key(field, item):
+    """Compute the sort key of item for field, mirroring what ItemFormatter displays."""
+    if field == "path":
+        return item.path
+    if field in ("type", "mode"):
+        mode = stat.filemode(item.get("mode", 0))
+        return mode[0] if field == "type" else mode
+    if field == "size":
+        return item.get_size()
+    if field in ("uid", "gid"):
+        value = item.get(field)
+        return -1 if value is None else value
+    if field in ("user", "group"):
+        value = item.get(field)
+        # if the name is not known, ItemFormatter displays the numeric id, so sort by that.
+        return str(item.get("uid" if field == "user" else "gid")) if value is None else value
+    if field in ("mtime", "ctime", "atime"):
+        # raw ns timestamps. ItemFormatter falls back to mtime for a missing atime/ctime, mirror that.
+        return item.get(field) or item.get("mtime") or 0
+    raise ValueError(f"Invalid sort field name: {field}")
+
 
 class ListMixIn:
     @with_repository(compatibility=(Manifest.Operation.READ,))
@@ -49,7 +77,8 @@ def item_filter(item):
                         return False
                 return True
 
-            for item in archive.iter_items(item_filter):
+            items = sorted_by_spec(archive.iter_items(item_filter), args.sort_by, item_sort_key)
+            for item in items:
                 sys.stdout.write(formatter.format_item(item, args.json_lines, sort=True))
 
         # Only load the cache if it will be used
@@ -103,6 +132,38 @@ def build_parser_list(self, subparsers, common_parser, mid_common_parser):
         """
             )
             + ItemFormatter.keys_help()
+            + textwrap.dedent(
+                """
+
+        Sorting
+        +++++++
+        Use ``--sort-by SPEC`` where SPEC is a comma-separated list of fields.
+        Sorts are applied stably from last to first in the given list, thus the first
+        field is the primary sort criterion. Prepend ">" for descending, "<" (or no
+        prefix) for ascending order, e.g. ``--sort-by='>size,path'`` (quote it, ">" is
+        a shell redirection).
+
+        Supported fields, sorting by the same value that ``--format`` would print:
+
+        - path: the item path
+        - type, mode: the file type character resp. the full file mode string
+        - user, group: the owner/group name, or the numeric id if the name is not known
+        - uid, gid: the numeric owner/group id
+        - size: the file size (0 for hardlink slaves, same as {size})
+        - mtime, ctime, atime: the timestamps (atime/ctime fall back to mtime if not stored)
+
+        There is no ``birthtime`` field, as there is no {birthtime} format key either.
+
+        The sort fields are independent of ``--format``, you may sort by a field you do
+        not print.
+
+        Note: ``--sort-by`` needs to read all (matching) items into memory before it can
+        print anything, so listing a huge archive this way needs a lot of memory and
+        delays the first output line. Without ``--sort-by``, items are streamed and their
+        order is undefined. If memory usage is a problem, consider
+        ``borg list --format ... | sort`` instead.
+        """
+            )
         )
         subparser = ArgumentParser(parents=[common_parser], description=self.do_list.__doc__, epilog=list_epilog)
         subparsers.add_subcommand("list", subparser, help="list archive contents")
@@ -128,6 +189,14 @@ def build_parser_list(self, subparsers, common_parser, mid_common_parser):
         subparser.add_argument(
             "--depth", metavar="N", dest="depth", type=int, help="only list files up to the specified directory depth"
         )
+        subparser.add_argument(
+            "--sort-by",
+            metavar="SPEC",
+            dest="sort_by",
+            type=item_sort_spec,
+            action=Highlander,
+            help="sort output by comma-separated fields (e.g. '>size,path'), see Sorting below",
+        )
         subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name")
         subparser.add_argument(
             "paths", metavar="PATH", nargs="*", type=PathSpec, help="paths to list; patterns are supported"

From 78fec4ee3190cb8218b7c459f3e1e105b5ce54b6 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 15:59:28 +0200
Subject: [PATCH 099/144] tests: borg list --sort-by

---
 src/borg/testsuite/archiver/list_cmd_test.py | 116 +++++++++++++++++++
 1 file changed, 116 insertions(+)

diff --git a/src/borg/testsuite/archiver/list_cmd_test.py b/src/borg/testsuite/archiver/list_cmd_test.py
index a3055ebc2d..2b31c38c83 100644
--- a/src/borg/testsuite/archiver/list_cmd_test.py
+++ b/src/borg/testsuite/archiver/list_cmd_test.py
@@ -4,6 +4,7 @@
 
 from ...constants import *  # NOQA
 from ...helpers import CommandError
+from .. import granularity_sleep
 from . import cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION, requires_hardlinks
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary")  # NOQA
@@ -299,3 +300,118 @@ def test_fingerprint(archivers, request):
 
     # Even unmodified files should have different fingerprints because conditions_hash changed
     assert fingerprints1["input/file2"] != fingerprints5["input/file2"]
+
+
+def test_list_sort_by_path(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "c_file", size=1)
+    create_regular_file(archiver.input_path, "a_file", size=1)
+    create_regular_file(archiver.input_path, "b_file", size=1)
+    cmd(archiver, "create", "test", "input")
+
+    expected = ["input", "input/a_file", "input/b_file", "input/c_file"]
+    assert cmd(archiver, "list", "test", "--short", "--sort-by=path").splitlines() == expected
+    assert cmd(archiver, "list", "test", "--short", "--sort-by=>path").splitlines() == list(reversed(expected))
+
+
+def test_list_sort_by_size_then_path(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "a_file", size=30)
+    create_regular_file(archiver.input_path, "b_file", size=10)
+    create_regular_file(archiver.input_path, "c_file", size=20)
+    cmd(archiver, "create", "test", "input")
+
+    output = cmd(archiver, "list", "test", "--sort-by=>size,path", "--format={size} {path}{NL}")
+    assert output.splitlines() == ["30 input/a_file", "20 input/c_file", "10 input/b_file", "0 input"]
+
+
+def test_list_sort_by_type(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "dir/file", size=1)
+    cmd(archiver, "create", "test", "input")
+
+    output = cmd(archiver, "list", "test", "--sort-by=type,path", "--format={type} {path}{NL}")
+    # regular files ("-") sort before directories ("d")
+    assert output.splitlines() == ["- input/dir/file", "d input", "d input/dir"]
+
+
+def test_list_sort_by_is_stable(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "a_file", size=10)
+    create_regular_file(archiver.input_path, "b_file", size=10)
+    cmd(archiver, "create", "test", "input")
+
+    # equal sizes, so the last sort field decides the order of these 2 items
+    output = cmd(archiver, "list", "test", "--short", "--sort-by=size,>path")
+    assert output.splitlines() == ["input", "input/b_file", "input/a_file"]
+
+
+@pytest.mark.parametrize("sort_spec", ["not_a_field", "path,not_a_field", "", ",", ">", "birthtime"])
+def test_list_sort_by_invalid_spec_is_rejected(archivers, request, sort_spec):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file", size=1)
+    cmd(archiver, "create", "test", "input")
+
+    cmd(archiver, "list", "test", f"--sort-by={sort_spec}", exit_code=EXIT_ERROR)
+
+
+def test_list_sort_by_twice_is_rejected(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file", size=1)
+    cmd(archiver, "create", "test", "input")
+
+    cmd(archiver, "list", "test", "--sort-by=path", "--sort-by=size", exit_code=EXIT_ERROR)
+
+
+@pytest.mark.parametrize(
+    "sort_key", ["path", "type", "mode", "user", "uid", "group", "gid", "size", "mtime", "ctime", "atime"]
+)
+def test_list_sort_by_all_keys_with_directions(archivers, request, sort_key):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "a_file", size=11)
+    create_regular_file(archiver.input_path, "b_file", size=22)
+    granularity_sleep()  # the files created below shall have newer timestamps
+    create_regular_file(archiver.input_path, "c_file", size=33)
+    create_regular_file(archiver.input_path, "dir/d_file", size=44)
+    cmd(archiver, "create", "test", "input")
+
+    expected_paths = {"input", "input/a_file", "input/b_file", "input/c_file", "input/dir", "input/dir/d_file"}
+    # We do not check the order here, this is mostly for coverage of all the sort keys.
+    for direction in ("<", ">"):
+        output = cmd(archiver, "list", "test", "--short", f"--sort-by={direction}{sort_key},path")
+        assert set(output.splitlines()) == expected_paths
+
+
+def test_list_sort_by_json_lines(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "a_file", size=1)
+    create_regular_file(archiver.input_path, "b_file", size=1)
+    cmd(archiver, "create", "test", "input")
+
+    output = cmd(archiver, "list", "test", "--json-lines", "--sort-by=>path")
+    paths = [json.loads(line)["path"] for line in output.splitlines()]
+    assert paths == ["input/b_file", "input/a_file", "input"]
+
+
+def test_list_sort_by_with_depth_and_pattern(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "b_file", size=1)
+    create_regular_file(archiver.input_path, "a_file", size=1)
+    create_regular_file(archiver.input_path, "dir/c_file", size=1)
+    cmd(archiver, "create", "test", "input")
+
+    # filtering happens before sorting
+    output = cmd(archiver, "list", "test", "--short", "--depth=1", "--sort-by=path")
+    assert output.splitlines() == ["input", "input/a_file", "input/b_file", "input/dir"]
+
+    output = cmd(archiver, "list", "test", "--short", "--sort-by=>path", "--pattern=+ re:_file$", "--pattern=- re:^.*$")
+    assert output.splitlines() == ["input/dir/c_file", "input/b_file", "input/a_file"]

From 465e9fa22a47f21ffbd298d73a7c7272d2caab7e Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 15:59:40 +0200
Subject: [PATCH 100/144] completion: complete the --sort-by fields of list and
 diff

There are 3 kinds of --sort-by now (archives, items, item diffs), each with
its own set of valid sort keys, so the completion helper is generated once per
key set from one template per shell. diff --sort-by had no completion at all
before.

tcsh matches completion rules by option name only, in one flat namespace for
all subcommands, so there it stays one helper offering the union of all keys.
---
 docs/usage/completion.rst.inc                 |   8 +-
 src/borg/archiver/completion_cmd.py           | 270 +++++++++++-------
 .../testsuite/archiver/completion_cmd_test.py | 134 ++++++++-
 3 files changed, 288 insertions(+), 124 deletions(-)

diff --git a/docs/usage/completion.rst.inc b/docs/usage/completion.rst.inc
index 9ce0720d3e..fe0dfb1936 100644
--- a/docs/usage/completion.rst.inc
+++ b/docs/usage/completion.rst.inc
@@ -47,4 +47,10 @@ This command prints a shell completion script for the given shell.
 Please note that for some dynamic completions (like archive IDs), the shell
 completion script will call borg to query the repository. This will work best
 if that call can be made without prompting for user input, so you may want to
-set BORG_REPO and BORG_PASSPHRASE environment variables.
\ No newline at end of file
+set BORG_REPO and BORG_PASSPHRASE environment variables.
+
+In tcsh, completions for positional arguments are matched by word position, so
+e.g. an archive name is only completed if no options precede it - one more
+reason to use BORG_REPO there rather than --repo. Also, options are matched by
+name only, so --sort-by offers the union of the sort keys valid for list, diff
+and the commands filtering archives.
\ No newline at end of file
diff --git a/src/borg/archiver/completion_cmd.py b/src/borg/archiver/completion_cmd.py
index e3d42c5442..1846c3567a 100644
--- a/src/borg/archiver/completion_cmd.py
+++ b/src/borg/archiver/completion_cmd.py
@@ -60,6 +60,8 @@
 import shtab
 
 from ._common import process_epilog
+from .diff_cmd import DIFF_SORT_KEYS, diff_sort_spec
+from .list_cmd import ITEM_SORT_KEYS, item_sort_spec
 from ..constants import *  # NOQA
 from ..helpers import (
     archivename_validator,
@@ -236,52 +238,7 @@
   compgen -W "${choices}" -- "$1"
 }
 
-# Complete comma-separated sort keys for any option with type=SortBySpec.
-# Keys are validated against Borg's AI_HUMAN_SORT_KEYS.
-_borg_complete_sortby() {
-  local cur="${COMP_WORDS[COMP_CWORD]}"
-
-  # Extract value part for --opt=value forms; otherwise the value is the word itself
-  local val prefix_eq
-  if [[ "$cur" == *=* ]]; then
-    prefix_eq="${cur%%=*}="
-    val="${cur#*=}"
-  else
-    prefix_eq=""
-    val="$cur"
-  fi
-
-  # Split into head (selected keys + trailing comma if any) and fragment (last token being typed)
-  local head frag
-  if [[ "$val" == *,* ]]; then
-    head="${val%,*},"
-    frag="${val##*,}"
-  else
-    head=""
-    frag="$val"
-  fi
-
-  # Build a comma-delimited list for cheap membership testing
-  local headlist
-  if [[ -n "$head" ]]; then
-    headlist=",${head%,},"
-  else
-    headlist=","  # nothing selected yet
-  fi
-
-  # Valid keys (embedded at generation time)
-  local keys=({SORT_KEYS})
-
-  local k
-  for k in "${keys[@]}"; do
-    # skip already-selected keys
-    [[ "$headlist" == *",${k},"* ]] && continue
-    # match prefix of last fragment
-    [[ -n "$frag" && "$k" != "$frag"* ]] && continue
-    printf '%s\n' "${prefix_eq}${head}${k}"
-  done
-}
-
+{SORTBY_FUNCS}
 # Complete comma-separated files cache mode tokens for options with type=FilesCacheMode.
 _borg_complete_filescachemode() {
   local cur="${COMP_WORDS[COMP_CWORD]}"
@@ -527,50 +484,7 @@
   compadd -V 'file size' -Q -a choices
 }
 
-# Complete comma-separated sort keys for any option with type=SortBySpec.
-_borg_complete_sortby() {
-  local cur
-  cur="${words[$CURRENT]}"
-
-  local val prefix_eq
-  if [[ "$cur" == *"="* ]]; then
-    prefix_eq="${cur%%\=*}="
-    val="${cur#*=}"
-  else
-    prefix_eq=""
-    val="$cur"
-  fi
-
-  local head frag
-  if [[ "$val" == *","* ]]; then
-    head="${val%,*},"
-    frag="${val##*,}"
-  else
-    head=""
-    frag="$val"
-  fi
-
-  local headlist
-  if [[ -n "$head" ]]; then
-    headlist=",${head%,},"
-  else
-    headlist=","  # nothing selected yet
-  fi
-
-  # Valid keys (embedded at generation time)
-  local -a keys=({SORT_KEYS})
-
-  local -a candidates=()
-  local k
-  for k in ${keys[@]}; do
-    [[ "$headlist" == *",${k},"* ]] && continue
-    [[ -n "$frag" && "$k" != "$frag"* ]] && continue
-    candidates+=( "${prefix_eq}${head}${k}" )
-  done
-  compadd -Q -- $candidates
-  return 0
-}
-
+{SORTBY_FUNCS}
 # Complete comma-separated files cache mode tokens for options with type=FilesCacheMode.
 _borg_complete_filescachemode() {
   local cur
@@ -753,17 +667,7 @@
     end
 end
 
-# Complete comma-separated sort keys for any option with type=SortBySpec.
-function _borg_complete_sortby
-    _borg_csv_scan
-    for k in {SORT_KEYS}
-        if contains -- $k $_borg_csv_selected
-            continue
-        end
-        _borg_csv_emit $k
-    end
-end
-
+{SORTBY_FUNCS}
 # Complete comma-separated files cache mode tokens for options with type=FilesCacheMode.
 function _borg_complete_filescachemode
     _borg_csv_scan
@@ -799,7 +703,7 @@
 alias _borg_complete_timestamp 'date +"%Y-%m-%dT%H:%M:%S"'
 
 
-alias _borg_complete_sortby "echo {SORT_KEYS}"
+{SORTBY_FUNCS}
 alias _borg_complete_filescachemode "echo {FCM_KEYS}"
 alias _borg_help_topics "echo {HELP_CHOICES}"
 alias _borg_complete_compression_spec "echo {COMP_SPEC_CHOICES}"
@@ -846,6 +750,149 @@
 )
 
 
+# There is more than one kind of --sort-by (archives, items, item diffs), each with its own set of
+# valid sort keys. The completion code is the same for all of them - only the helper's name and the
+# key list differ - so it is generated from one template per shell, rendered once per key set, see
+# SORTBY_COMPLETERS and _sortby_functions.
+BASH_SORTBY_FN_TMPL = r"""
+# Complete comma-separated sort keys for the options using this set of keys.
+{FN_NAME}() {
+  local cur="${COMP_WORDS[COMP_CWORD]}"
+
+  # Extract value part for --opt=value forms; otherwise the value is the word itself
+  local val prefix_eq
+  if [[ "$cur" == *=* ]]; then
+    prefix_eq="${cur%%=*}="
+    val="${cur#*=}"
+  else
+    prefix_eq=""
+    val="$cur"
+  fi
+
+  # Split into head (selected keys + trailing comma if any) and fragment (last token being typed)
+  local head frag
+  if [[ "$val" == *,* ]]; then
+    head="${val%,*},"
+    frag="${val##*,}"
+  else
+    head=""
+    frag="$val"
+  fi
+
+  # Build a comma-delimited list for cheap membership testing
+  local headlist
+  if [[ -n "$head" ]]; then
+    headlist=",${head%,},"
+  else
+    headlist=","  # nothing selected yet
+  fi
+
+  # Valid keys (embedded at generation time)
+  local keys=({KEYS})
+
+  local k
+  for k in "${keys[@]}"; do
+    # skip already-selected keys
+    [[ "$headlist" == *",${k},"* ]] && continue
+    # match prefix of last fragment
+    [[ -n "$frag" && "$k" != "$frag"* ]] && continue
+    printf '%s\n' "${prefix_eq}${head}${k}"
+  done
+}
+"""
+
+ZSH_SORTBY_FN_TMPL = r"""
+# Complete comma-separated sort keys for the options using this set of keys.
+{FN_NAME}() {
+  local cur
+  cur="${words[$CURRENT]}"
+
+  local val prefix_eq
+  if [[ "$cur" == *"="* ]]; then
+    prefix_eq="${cur%%\=*}="
+    val="${cur#*=}"
+  else
+    prefix_eq=""
+    val="$cur"
+  fi
+
+  local head frag
+  if [[ "$val" == *","* ]]; then
+    head="${val%,*},"
+    frag="${val##*,}"
+  else
+    head=""
+    frag="$val"
+  fi
+
+  local headlist
+  if [[ -n "$head" ]]; then
+    headlist=",${head%,},"
+  else
+    headlist=","  # nothing selected yet
+  fi
+
+  # Valid keys (embedded at generation time)
+  local -a keys=({KEYS})
+
+  local -a candidates=()
+  local k
+  for k in ${keys[@]}; do
+    [[ "$headlist" == *",${k},"* ]] && continue
+    [[ -n "$frag" && "$k" != "$frag"* ]] && continue
+    candidates+=( "${prefix_eq}${head}${k}" )
+  done
+  compadd -Q -- $candidates
+  return 0
+}
+"""
+
+FISH_SORTBY_FN_TMPL = r"""
+# Complete comma-separated sort keys for the options using this set of keys.
+function {FN_NAME}
+    _borg_csv_scan
+    for k in {KEYS}
+        if contains -- $k $_borg_csv_selected
+            continue
+        end
+        _borg_csv_emit $k
+    end
+end
+"""
+
+TCSH_SORTBY_FN_TMPL = 'alias {FN_NAME} "echo {KEYS}"\n'
+
+SORTBY_FN_TMPLS = {
+    "bash": BASH_SORTBY_FN_TMPL,
+    "zsh": ZSH_SORTBY_FN_TMPL,
+    "fish": FISH_SORTBY_FN_TMPL,
+    "tcsh": TCSH_SORTBY_FN_TMPL,
+}
+
+# tcsh matches completion rules by option name only, in one flat namespace for all subcommands,
+# so it can not tell `borg list --sort-by` from `borg repo-list --sort-by`. Thus, it gets a single
+# helper offering the union of all sort keys, see for_all_shells(tcsh_fn=...).
+TCSH_SORTBY_FN = "_borg_complete_sortby"
+
+# the --sort-by flavours: (argparse type, name of the shell helper completing it, valid sort keys)
+SORTBY_COMPLETERS = [
+    (SortBySpec, "_borg_complete_sortby", AI_HUMAN_SORT_KEYS),  # archives: repo-list, ...
+    (item_sort_spec, "_borg_complete_item_sortby", ITEM_SORT_KEYS),  # items: list
+    (diff_sort_spec, "_borg_complete_diff_sortby", DIFF_SORT_KEYS),  # item diffs: diff
+]
+
+
+def _sortby_functions(shell, completers):
+    """Render the --sort-by completion helpers for shell, one per set of sort keys."""
+    template = SORTBY_FN_TMPLS[shell]
+    if shell == "tcsh":
+        union = sorted({key for _, _, keys in completers for key in keys})
+        completers = [(None, TCSH_SORTBY_FN, union)]
+    return "".join(
+        partial_format(template, {"FN_NAME": fn_name, "KEYS": " ".join(keys)}) for _, fn_name, keys in completers
+    )
+
+
 # in a `p@N@`...`@` rule, one `if (...) <action>` clause, e.g.
 # `if ( $#cmd >= 3 && ("$cmd[2]" == "key") && ("$cmd[3]" == "export") ) f`
 TCSH_CLAUSE_RE = re.compile(r"if \( \$#cmd >= \d+ && (?P<checks>.*?) \) (?P<action>.*)")
@@ -938,13 +985,16 @@ def do_completion(self, args):
         finally:
             self.prog = prog
 
-        def for_all_shells(fn_name):
+        def for_all_shells(fn_name, tcsh_fn=None):
             # same-named completion helper in the bash, zsh, tcsh and fish preambles;
-            # fish needs a command substitution to call it, tcsh backquotes
-            return {"bash": fn_name, "zsh": fn_name, "tcsh": f"`{fn_name}`", "fish": f"({fn_name})"}
+            # fish needs a command substitution to call it, tcsh backquotes.
+            # tcsh matches rules by option name only, over all subcommands, so options needing
+            # different completions per subcommand have to share one tcsh helper.
+            return {"bash": fn_name, "zsh": fn_name, "tcsh": f"`{tcsh_fn or fn_name}`", "fish": f"({fn_name})"}
 
         _attach_completion(parser, archivename_validator, for_all_shells("_borg_complete_archive"))
-        _attach_completion(parser, SortBySpec, for_all_shells("_borg_complete_sortby"))
+        for sortby_type, sortby_fn, _ in SORTBY_COMPLETERS:
+            _attach_completion(parser, sortby_type, for_all_shells(sortby_fn, tcsh_fn=TCSH_SORTBY_FN))
         _attach_completion(parser, FilesCacheMode, for_all_shells("_borg_complete_filescachemode"))
         _attach_completion(parser, CompressionSpec, for_all_shells("_borg_complete_compression_spec"))
         _attach_completion(parser, PathSpec, shtab.DIRECTORY)
@@ -965,7 +1015,6 @@ def for_all_shells(fn_name):
         _attach_help_completion(parser, for_all_shells("_borg_help_topics"))
 
         # Build preambles using partial_format to avoid escaping braces etc.
-        sort_keys = " ".join(AI_HUMAN_SORT_KEYS)
         fcm_keys = " ".join(["ctime", "mtime", "size", "inode", "rechunk", "disabled"])  # keep in sync with parser
 
         # Help completion templates
@@ -1006,7 +1055,6 @@ def for_all_shells(fn_name):
         file_size_choices_str = " ".join(file_size_choices)
 
         mapping = {
-            "SORT_KEYS": sort_keys,
             "FCM_KEYS": fcm_keys,
             "COMP_SPEC_CHOICES": comp_spec_choices_str,
             "CHUNKER_PARAMS_CHOICES": chunker_params_choices_str,
@@ -1014,6 +1062,8 @@ def for_all_shells(fn_name):
             "FILE_SIZE_CHOICES": file_size_choices_str,
             "HELP_CHOICES": help_choices,
             "SH_COMPLETE": TCSH_SH_COMPLETE,
+            # last, so that the rendered helpers are not scanned for the placeholders above
+            "SORTBY_FUNCS": _sortby_functions(args.shell, SORTBY_COMPLETERS) if args.shell in SORTBY_FN_TMPLS else "",
         }
         preamble_templates = {
             "bash": BASH_PREAMBLE_TMPL,
@@ -1043,7 +1093,9 @@ def build_parser_completion(self, subparsers, common_parser, mid_common_parser):
 
         In tcsh, completions for positional arguments are matched by word position, so
         e.g. an archive name is only completed if no options precede it - one more
-        reason to use BORG_REPO there rather than --repo.
+        reason to use BORG_REPO there rather than --repo. Also, options are matched by
+        name only, so --sort-by offers the union of the sort keys valid for list, diff
+        and the commands filtering archives.
         """
         )
 
diff --git a/src/borg/testsuite/archiver/completion_cmd_test.py b/src/borg/testsuite/archiver/completion_cmd_test.py
index 2260f8fb50..171806cc45 100644
--- a/src/borg/testsuite/archiver/completion_cmd_test.py
+++ b/src/borg/testsuite/archiver/completion_cmd_test.py
@@ -8,6 +8,7 @@
 import pytest
 
 import borg
+from ...archiver.completion_cmd import TCSH_SORTBY_FN
 from . import cmd, generate_archiver_tests, RK_ENCRYPTION
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local")  # NOQA
@@ -58,6 +59,28 @@ def _run_bash_completion_fn(completion_script, setup_code):
     return result
 
 
+def _run_zsh_completion_fn(completion_script, setup_code):
+    """Run a preamble helper of the completion script in zsh, return subprocess result.
+
+    The generated script can not just be sourced: its tail calls _shtab_borg when it is not
+    sourced from a completion context. So we only take the preamble (shtab wraps it into
+    markers) and stub compadd, which is how the helpers report their candidates.
+    """
+    preamble = completion_script.split("# Custom Preamble\n", 1)[1].split("\n# End Custom Preamble", 1)[0]
+    code = (
+        preamble + "\n"
+        'compadd() { local a; for a in "$@"; do [[ $a == -* ]] && continue; print -r -- $a; done }\n' + setup_code
+    )
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".zsh", delete=False) as f:
+        f.write(code)
+        script_path = f.name
+    try:
+        # -f: do not read the startup files
+        return subprocess.run(["zsh", "-f", script_path], capture_output=True, text=True, timeout=120)
+    finally:
+        os.unlink(script_path)
+
+
 def _fish_quote(text):
     """Escape text for inclusion in a single-quoted fish string."""
     return text.replace("\\", "\\\\").replace("'", "\\'")
@@ -217,27 +240,110 @@ def test_tcsh_completion_syntax(archivers, request):
     assert result.returncode == 0, f"Generated Tcsh completion has errors: {result.stderr.decode()}"
 
 
-# -- borg-specific preamble function behavior (bash) --------------------------
+# -- --sort-by completion (one helper per set of sort keys) -------------------
+
+# (command, name of the shell helper completing its --sort-by)
+SORTBY_WIRING = [
+    ("repo-list", "_borg_complete_sortby"),
+    ("list", "_borg_complete_item_sortby"),
+    ("diff", "_borg_complete_diff_sortby"),
+]
+
+# (helper, already typed value, keys it must offer, keys it must not offer)
+SORTBY_BEHAVIOR = [
+    # archive sort keys, must not re-offer an already selected key
+    ("_borg_complete_sortby", "timestamp,", ["timestamp,archive"], ["timestamp,timestamp"]),
+    # item sort keys, not the archive or item diff ones
+    ("_borg_complete_item_sortby", "", ["path", "size", "atime"], ["timestamp", "archive", "size_added"]),
+    ("_borg_complete_item_sortby", "size,", ["size,path"], ["size,size"]),
+    # item diff sort keys: "size" is a valid key, but does not match the "size_" fragment
+    ("_borg_complete_diff_sortby", "size_", ["size_added", "size_removed", "size_diff"], ["size"]),
+    ("_borg_complete_diff_sortby", "path,", ["path,size_added"], ["path,path", "path,atime"]),
+]
+
+
+@pytest.mark.parametrize("command,fn", SORTBY_WIRING)
+def test_bash_sortby_wiring(archivers, request, command, fn):
+    output = cmd(request.getfixturevalue(archivers), "completion", "bash")
+    assert f"_shtab_borg_{command.replace('-', '_')}___sort_by_COMPGEN={fn}\n" in output
+    assert f"\n{fn}() {{\n" in output
+
+
+@pytest.mark.parametrize("command,fn", SORTBY_WIRING)
+def test_zsh_sortby_wiring(archivers, request, command, fn):
+    output = cmd(request.getfixturevalue(archivers), "completion", "zsh")
+    block = output.split(f"\n_shtab_borg_{command.replace('-', '_')}_options=(\n", 1)[1].split("\n)\n", 1)[0]
+    assert f':sort_by:{fn}"' in block
+    assert f"\n{fn}() {{\n" in output
+
+
+@pytest.mark.parametrize("command,fn", SORTBY_WIRING)
+def test_fish_sortby_wiring(archivers, request, command, fn):
+    output = cmd(request.getfixturevalue(archivers), "completion", "fish")
+    assert any(
+        f"_shtab_borg_using {command}'" in line and " -l sort-by " in line and f'"({fn})"' in line
+        for line in output.splitlines()
+    )
+    assert f"\nfunction {fn}\n" in output
+
+
+def test_tcsh_sortby_single_rule(archivers, request):
+    """tcsh matches by option name only, so all --sort-by options share one helper."""
+    output = cmd(request.getfixturevalue(archivers), "completion", "tcsh")
+    assert output.count("'n/--sort-by/") == 1
+    assert f"'n/--sort-by/`{TCSH_SORTBY_FN}`/'" in output
+    alias = next(line for line in output.splitlines() if line.startswith(f"alias {TCSH_SORTBY_FN} "))
+    for key in ("timestamp", "atime", "size_added"):  # the union of all the sort keys
+        assert f" {key}" in alias
+
+
+@pytest.mark.parametrize("shell", ["bash", "zsh", "fish", "tcsh"])
+def test_completion_helpers_are_defined(archivers, request, shell):
+    """Every _borg_* helper the generated script refers to must also be defined by it."""
+    output = cmd(request.getfixturevalue(archivers), "completion", shell)
+    defined = set(re.findall(r"^(?:function |alias )?(_borg_\w+)\s*(?:\(\)|\(|\s|$)", output, re.M))
+    referenced = set(re.findall(r"_borg_complete_\w+|_borg_help_topics", output))
+    assert referenced <= defined, f"undefined: {sorted(referenced - defined)}"
 
 
 @needs_bash
-def test_bash_sortby_dedup(archivers, request):
-    """_borg_complete_sortby should not re-offer already-selected sort keys."""
+@pytest.mark.parametrize("fn,typed,expected,not_expected", SORTBY_BEHAVIOR)
+def test_bash_sortby_keysets(archivers, request, fn, typed, expected, not_expected):
     archiver = request.getfixturevalue(archivers)
     script = cmd(archiver, "completion", "bash")
 
-    # Simulate: user typed "borg repo-list --sort-by timestamp,"
-    # The function should offer remaining keys but NOT "timestamp" again.
-    result = _run_bash_completion_fn(
-        script, 'COMP_WORDS=(borg repo-list --sort-by "timestamp,")\n' "COMP_CWORD=3\n" "_borg_complete_sortby\n"
-    )
+    result = _run_bash_completion_fn(script, f'COMP_WORDS=(borg cmd --sort-by "{typed}")\nCOMP_CWORD=3\n{fn}\n')
     assert result.returncode == 0, f"stderr: {result.stderr}"
-    lines = [line for line in result.stdout.strip().splitlines() if line.strip()]
-    # "timestamp" must not appear as a standalone completion candidate
-    bare_keys = [line.rsplit(",", 1)[-1] for line in lines]
-    assert "timestamp" not in bare_keys, f"timestamp was re-offered: {lines}"
-    # Other keys like "archive" should be offered
-    assert any("archive" in line for line in lines), f"expected 'archive' in completions: {lines}"
+    candidates = result.stdout.split()
+    assert set(expected) <= set(candidates), f"missing from {candidates}"
+    assert not set(not_expected) & set(candidates), f"unexpected in {candidates}"
+
+
+@needs_zsh
+@pytest.mark.parametrize("fn,typed,expected,not_expected", SORTBY_BEHAVIOR)
+def test_zsh_sortby_keysets(archivers, request, fn, typed, expected, not_expected):
+    archiver = request.getfixturevalue(archivers)
+    script = cmd(archiver, "completion", "zsh")
+
+    result = _run_zsh_completion_fn(script, f"words=(borg cmd --sort-by '{typed}')\nCURRENT=4\n{fn}\n")
+    assert result.returncode == 0, f"stderr: {result.stderr}"
+    candidates = result.stdout.split()
+    assert set(expected) <= set(candidates), f"missing from {candidates}"
+    assert not set(not_expected) & set(candidates), f"unexpected in {candidates}"
+
+
+@needs_fish
+def test_fish_sortby_keysets(archivers, request):
+    """Each command's --sort-by completes its own sort keys."""
+    archiver = request.getfixturevalue(archivers)
+    script = cmd(archiver, "completion", "fish")
+
+    items = _fish_complete_candidates(script, "borg list --sort-by ")
+    assert "atime" in items and "timestamp" not in items and "size_added" not in items
+    diffs = _fish_complete_candidates(script, "borg diff --sort-by size_added,")
+    assert "size_added,path" in diffs and "size_added,size_added" not in diffs
+    archives = _fish_complete_candidates(script, "borg repo-list --sort-by ")
+    assert "timestamp" in archives and "atime" not in archives
 
 
 @needs_bash

From fa66d83461d21a42f29c625c6797faa9732dcc75 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 15:59:52 +0200
Subject: [PATCH 101/144] docs: list --sort-by

---
 docs/changes.rst        |  1 +
 docs/usage/list.rst     |  9 +++++++++
 docs/usage/list.rst.inc | 38 +++++++++++++++++++++++++++++++++++---
 3 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index f51e34bb67..0ac7066af3 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -222,6 +222,7 @@ New features:
   - use --chunking / --hashing / --encrypting / --compressing / --msgpacking
     to run only a subset of the benchmarks, #10050
 - completion: generate fish and tcsh completions, #9989, #9503
+- list: --sort-by=field[,field,...], #9009
 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513
 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041
 
diff --git a/docs/usage/list.rst b/docs/usage/list.rst
index e0a9b7ca2f..34610a3a57 100644
--- a/docs/usage/list.rst
+++ b/docs/usage/list.rst
@@ -36,3 +36,12 @@ Examples
     -rw-rw-r-- user   user    1416192 Sun, 2015-02-01 11:00:00 code/myproject/file.text
     ...
 
+    # Use --sort-by with a comma-separated list; sorts apply stably from last to first.
+    # Here: primary by size descending, tie-breaker by path ascending.
+    # Note the quoting - ">" would otherwise be a shell redirection.
+    $ borg list archiveA --sort-by='>size,path' --format='{size:8d} {path}{NL}'
+     1416192 code/myproject/file.ext
+     1416192 code/myproject/file.text
+           0 .
+    ...
+
diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc
index 761f1021be..bf682d5848 100644
--- a/docs/usage/list.rst.inc
+++ b/docs/usage/list.rst.inc
@@ -29,6 +29,8 @@ borg list
     +-------------------------------------------------------+---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |                                                       | ``--depth N``                         | only list files up to the specified directory depth                                                                                                                                   |
     +-------------------------------------------------------+---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--sort-by SPEC``                    | sort output by comma-separated fields (e.g. '>size,path'), see Sorting below                                                                                                          |
+    +-------------------------------------------------------+---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     | .. class:: borg-common-opt-ref                                                                                                                                                                                                                                                        |
     |                                                                                                                                                                                                                                                                                       |
     | :ref:`common_options`                                                                                                                                                                                                                                                                 |
@@ -61,10 +63,11 @@ borg list
 
 
     options
-        --short     only print file/directory names, nothing else
+        --short            only print file/directory names, nothing else
         --format FORMAT    specify format for file listing (default: "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}")
-        --json-lines    Format output as JSON Lines. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text.
-        --depth N    only list files up to the specified directory depth
+        --json-lines       Format output as JSON Lines. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text.
+        --depth N          only list files up to the specified directory depth
+        --sort-by SPEC     sort output by comma-separated fields (e.g. '>size,path'), see Sorting below
 
 
     :ref:`common_options`
@@ -157,3 +160,32 @@ Keys available only when listing files in an archive:
 - archiveid: internal ID of the archive
 - archivename: name of the archive
 - extra: prepends {target} with " -> " for soft links and " link to " for hard links
+
+
+Sorting
++++++++
+Use ``--sort-by SPEC`` where SPEC is a comma-separated list of fields.
+Sorts are applied stably from last to first in the given list, thus the first
+field is the primary sort criterion. Prepend ">" for descending, "<" (or no
+prefix) for ascending order, e.g. ``--sort-by='>size,path'`` (quote it, ">" is
+a shell redirection).
+
+Supported fields, sorting by the same value that ``--format`` would print:
+
+- path: the item path
+- type, mode: the file type character resp. the full file mode string
+- user, group: the owner/group name, or the numeric id if the name is not known
+- uid, gid: the numeric owner/group id
+- size: the file size (0 for hardlink slaves, same as {size})
+- mtime, ctime, atime: the timestamps (atime/ctime fall back to mtime if not stored)
+
+There is no ``birthtime`` field, as there is no {birthtime} format key either.
+
+The sort fields are independent of ``--format``, you may sort by a field you do
+not print.
+
+Note: ``--sort-by`` needs to read all (matching) items into memory before it can
+print anything, so listing a huge archive this way needs a lot of memory and
+delays the first output line. Without ``--sort-by``, items are streamed and their
+order is undefined. If memory usage is a problem, consider
+``borg list --format ... | sort`` instead.

From d5eabd2b112d4dfcdfc77120dde2207c05137843 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 16:00:57 +0200
Subject: [PATCH 102/144] diff: --sort-by can only be given once

Like list --sort-by, repo-list --sort-by and diff --format.
---
 src/borg/archiver/diff_cmd.py                |  1 +
 src/borg/testsuite/archiver/diff_cmd_test.py | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/src/borg/archiver/diff_cmd.py b/src/borg/archiver/diff_cmd.py
index 30e378bf28..22ac587b70 100644
--- a/src/borg/archiver/diff_cmd.py
+++ b/src/borg/archiver/diff_cmd.py
@@ -293,6 +293,7 @@ def build_parser_diff(self, subparsers, common_parser, mid_common_parser):
             "--sort-by",
             dest="sort_by",
             type=diff_sort_spec,
+            action=Highlander,
             help="Sort output by comma-separated fields (e.g., '>size_added,path').",
         )
         subparser.add_argument(
diff --git a/src/borg/testsuite/archiver/diff_cmd_test.py b/src/borg/testsuite/archiver/diff_cmd_test.py
index ab073bc8af..171a06cbd6 100644
--- a/src/borg/testsuite/archiver/diff_cmd_test.py
+++ b/src/borg/testsuite/archiver/diff_cmd_test.py
@@ -330,6 +330,17 @@ def test_sort_by_invalid_field_is_rejected(archivers, request):
     cmd(archiver, "diff", "a1", "a2", "--sort-by=not_a_field", exit_code=EXIT_ERROR)
 
 
+def test_sort_by_twice_is_rejected(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+
+    create_regular_file(archiver.input_path, "file", size=1)
+    cmd(archiver, "create", "a1", "input")
+    cmd(archiver, "create", "a2", "input")
+
+    cmd(archiver, "diff", "a1", "a2", "--sort-by=path", "--sort-by=size", exit_code=EXIT_ERROR)
+
+
 def test_sort_by_size_added_then_path(archivers, request):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "repo-create", RK_ENCRYPTION)

From 46cfdd8796668506d4917b489c91c117e4a1088e Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 16:42:06 +0200
Subject: [PATCH 103/144] tests: make the --sort-by completion wiring checks
 CRLF-proof

On Windows the captured output has CRLF line endings, so assertions anchored
on "\n" did not match. Match on lines instead.
---
 .../testsuite/archiver/completion_cmd_test.py | 39 +++++++++++--------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/src/borg/testsuite/archiver/completion_cmd_test.py b/src/borg/testsuite/archiver/completion_cmd_test.py
index 171806cc45..1ef5add254 100644
--- a/src/borg/testsuite/archiver/completion_cmd_test.py
+++ b/src/borg/testsuite/archiver/completion_cmd_test.py
@@ -66,7 +66,7 @@ def _run_zsh_completion_fn(completion_script, setup_code):
     sourced from a completion context. So we only take the preamble (shtab wraps it into
     markers) and stub compadd, which is how the helpers report their candidates.
     """
-    preamble = completion_script.split("# Custom Preamble\n", 1)[1].split("\n# End Custom Preamble", 1)[0]
+    preamble = completion_script.split("# Custom Preamble", 1)[1].split("# End Custom Preamble", 1)[0]
     code = (
         preamble + "\n"
         'compadd() { local a; for a in "$@"; do [[ $a == -* ]] && continue; print -r -- $a; done }\n' + setup_code
@@ -262,37 +262,42 @@ def test_tcsh_completion_syntax(archivers, request):
 ]
 
 
+def completion_lines(archivers, request, shell):
+    """The generated completion script as a list of lines (on Windows it has CRLF line endings)."""
+    return cmd(request.getfixturevalue(archivers), "completion", shell).splitlines()
+
+
 @pytest.mark.parametrize("command,fn", SORTBY_WIRING)
 def test_bash_sortby_wiring(archivers, request, command, fn):
-    output = cmd(request.getfixturevalue(archivers), "completion", "bash")
-    assert f"_shtab_borg_{command.replace('-', '_')}___sort_by_COMPGEN={fn}\n" in output
-    assert f"\n{fn}() {{\n" in output
+    lines = completion_lines(archivers, request, "bash")
+    assert f"_shtab_borg_{command.replace('-', '_')}___sort_by_COMPGEN={fn}" in lines
+    assert f"{fn}() {{" in lines
 
 
 @pytest.mark.parametrize("command,fn", SORTBY_WIRING)
 def test_zsh_sortby_wiring(archivers, request, command, fn):
-    output = cmd(request.getfixturevalue(archivers), "completion", "zsh")
-    block = output.split(f"\n_shtab_borg_{command.replace('-', '_')}_options=(\n", 1)[1].split("\n)\n", 1)[0]
-    assert f':sort_by:{fn}"' in block
-    assert f"\n{fn}() {{\n" in output
+    lines = completion_lines(archivers, request, "zsh")
+    block = lines[lines.index(f"_shtab_borg_{command.replace('-', '_')}_options=(") :]
+    block = block[: block.index(")")]
+    assert any(f':sort_by:{fn}"' in line for line in block)
+    assert f"{fn}() {{" in lines
 
 
 @pytest.mark.parametrize("command,fn", SORTBY_WIRING)
 def test_fish_sortby_wiring(archivers, request, command, fn):
-    output = cmd(request.getfixturevalue(archivers), "completion", "fish")
+    lines = completion_lines(archivers, request, "fish")
     assert any(
-        f"_shtab_borg_using {command}'" in line and " -l sort-by " in line and f'"({fn})"' in line
-        for line in output.splitlines()
+        f"_shtab_borg_using {command}'" in line and " -l sort-by " in line and f'"({fn})"' in line for line in lines
     )
-    assert f"\nfunction {fn}\n" in output
+    assert f"function {fn}" in lines
 
 
 def test_tcsh_sortby_single_rule(archivers, request):
     """tcsh matches by option name only, so all --sort-by options share one helper."""
-    output = cmd(request.getfixturevalue(archivers), "completion", "tcsh")
-    assert output.count("'n/--sort-by/") == 1
-    assert f"'n/--sort-by/`{TCSH_SORTBY_FN}`/'" in output
-    alias = next(line for line in output.splitlines() if line.startswith(f"alias {TCSH_SORTBY_FN} "))
+    lines = completion_lines(archivers, request, "tcsh")
+    assert sum(line.count("'n/--sort-by/") for line in lines) == 1
+    assert any(f"'n/--sort-by/`{TCSH_SORTBY_FN}`/'" in line for line in lines)
+    alias = next(line for line in lines if line.startswith(f"alias {TCSH_SORTBY_FN} "))
     for key in ("timestamp", "atime", "size_added"):  # the union of all the sort keys
         assert f" {key}" in alias
 
@@ -300,7 +305,7 @@ def test_tcsh_sortby_single_rule(archivers, request):
 @pytest.mark.parametrize("shell", ["bash", "zsh", "fish", "tcsh"])
 def test_completion_helpers_are_defined(archivers, request, shell):
     """Every _borg_* helper the generated script refers to must also be defined by it."""
-    output = cmd(request.getfixturevalue(archivers), "completion", shell)
+    output = "\n".join(completion_lines(archivers, request, shell))
     defined = set(re.findall(r"^(?:function |alias )?(_borg_\w+)\s*(?:\(\)|\(|\s|$)", output, re.M))
     referenced = set(re.findall(r"_borg_complete_\w+|_borg_help_topics", output))
     assert referenced <= defined, f"undefined: {sorted(referenced - defined)}"

From 0b451c1cb6af923aa0a558a6efe0a151f4239328 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 12 Aug 2026 17:17:34 +0200
Subject: [PATCH 104/144] CI: force colored tox/pytest output in all jobs

Only the Windows jobs had colored output (job-level PY_COLORS). Set
PY_COLORS=1 (pytest) and TOX_COLORED=yes (tox 4's own output) at the
workflow level in ci.yml and canary.yml - tox forwards both to pytest
via pass_env = ["*"]. Remove the now-redundant job-level PY_COLORS
from the Windows jobs.

The vm_tests job runs its script inside cross-platform-actions VMs
where the workflow-level env does not arrive, so export both vars in
the VM script, next to the existing PYTEST_TIMEOUT export.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/canary.yml |  8 +++++++-
 .github/workflows/ci.yml     | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml
index 36fcc0a796..b812d61542 100644
--- a/.github/workflows/canary.yml
+++ b/.github/workflows/canary.yml
@@ -8,6 +8,13 @@ on:
 permissions:
   contents: read
 
+env:
+  # Force colored tox and pytest output even without a tty - the GitHub
+  # Actions log viewer renders ANSI colors. tox passes both vars through
+  # to pytest (pass_env = ["*"]).
+  PY_COLORS: "1"       # pytest
+  TOX_COLORED: "yes"   # tox's own output
+
 jobs:
   canary_tests:
     name: Canary (${{ matrix.os }}, ${{ matrix.python-version }}, ${{ matrix.toxenv }})
@@ -107,7 +114,6 @@ jobs:
     timeout-minutes: 180
 
     env:
-      PY_COLORS: 1
       MSYS2_ARG_CONV_EXCL: "*"
       MSYS2_ENV_CONV_EXCL: "*"
       # see the "Cache pip-built wheels" step. MSYS2_ENV_CONV_EXCL above keeps
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ae34d02ad5..8de5fb3694 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -28,6 +28,13 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  # Force colored tox and pytest output even without a tty - the GitHub
+  # Actions log viewer renders ANSI colors. tox passes both vars through
+  # to pytest (pass_env = ["*"]).
+  PY_COLORS: "1"       # pytest
+  TOX_COLORED: "yes"   # tox's own output
+
 jobs:
   lint:
 
@@ -470,6 +477,11 @@ jobs:
           # job timeout (tox passes all env vars through to pytest).
           export PYTEST_TIMEOUT=300
 
+          # colored tox/pytest output - the workflow-level env vars do not
+          # reach into the VM, so export them here again.
+          export PY_COLORS=1
+          export TOX_COLORED=yes
+
           # see the "Cache pip-built wheels" step
           export PIP_CACHE_DIR="$PWD/.pip-cache"
 
@@ -668,7 +680,6 @@ jobs:
     needs: [lint]
 
     env:
-      PY_COLORS: 1
       MSYS2_ARG_CONV_EXCL: "*"
       MSYS2_ENV_CONV_EXCL: "*"
       # see the "Cache pip-built wheels" step. MSYS2_ENV_CONV_EXCL above keeps

From 0ccf95b431a6ea9b51903fd7366eca7b096f738c Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 12 Aug 2026 12:52:27 +0530
Subject: [PATCH 105/144] validate object headers when walking a pack, see
 #8476

PackReader.iter_headers took the next offset from the sizes in each object
header without ever checking that header. On corruption the walk either
continues on payload bytes and yields garbage (chunk_id, offset, size)
tuples, or skips past the end of the pack, where the short read looks like
a clean EOF. The index build_chunkindex_from_repo rebuilds from that is
wrong either way, without saying so.

Check OBJ_MAGIC and that the object fits into the pack, raise IntegrityError
naming the pack otherwise, like check_pack_objects does.

The bounds check needs the pack size, which PackReader did not have, so add
PackReader.size(): len(pack_contents) in memory, one store.info() per pack
otherwise. No per-object roundtrip is added.
---
 src/borg/repository.py                | 27 ++++++++++++++-
 src/borg/testsuite/repository_test.py | 49 +++++++++++++++++++++++++++
 2 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index 120a0f59ec..ba99358fb1 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -361,12 +361,28 @@ def read(self, offset, size):
             return memoryview(self.pack_contents)[offset : offset + size]
         return self.store.load(self.key, offset=offset, size=size)
 
+    def size(self):
+        """Return the pack size in bytes; for a store-backed pack this is one metadata lookup."""
+        if self.pack_contents is not None:
+            return len(self.pack_contents)
+        return self.store.info(self.key).size
+
     def iter_headers(self):
         """Yield (chunk_id, offset, size) for each object by walking the fixed object headers.
 
         Only the headers are read, not the payloads, so locating every object costs one short
-        range read per object (or just a slice, when the pack is already in memory).
+        range read per object (or just a slice, when the pack is already in memory), plus one
+        store metadata lookup for the pack size.
+
+        Each full header must have OBJ_MAGIC and describe an object that fits into the pack,
+        otherwise the pack is corrupt and IntegrityError is raised. Ending the walk instead
+        would be worse than raising: the chunks index rebuilt from these headers would just be
+        missing the rest of the pack, and borg check --repair would then "fix" the archives by
+        dropping chunks that are there.
+        A trailing partial header is the clean end of the pack, not corruption.
         """
+        pack_hex = bin_to_hex(self.pack_id) if self.pack_id is not None else "<no id>"
+        pack_size = self.size()
         hdr_size = RepoObj.obj_header.size
         offset = 0
         while True:
@@ -374,7 +390,16 @@ def iter_headers(self):
             if len(hdr_data) < hdr_size:
                 break  # clean EOF, or trailing partial bytes
             hdr = RepoObj.ObjHeader(*RepoObj.obj_header.unpack(hdr_data))
+            if hdr.magic != OBJ_MAGIC:
+                raise IntegrityError(
+                    f'pack {pack_hex}: no object header at offset {offset} (pack corruption), run "borg check"'
+                )
             obj_size = hdr_size + hdr.meta_size + hdr.data_size
+            if offset + obj_size > pack_size:
+                raise IntegrityError(
+                    f"pack {pack_hex}: object extends past end of file at offset {offset} "
+                    f'(pack corruption), run "borg check"'
+                )
             yield hdr.chunk_id, offset, obj_size
             offset += obj_size
 
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 34d39646fb..9e1abb1aac 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1635,6 +1635,55 @@ def test_pack_reader_iter_headers_reads_through_store(tmp_path):
         assert list(reader.iter_headers()) == [(H(47), 0, len(obj1)), (H(48), len(obj1), len(obj2))]
 
 
+def test_pack_reader_raises_on_bad_magic():
+    # a header without OBJ_MAGIC means the walk desynced onto payload bytes: corruption, not EOF.
+    obj1 = fchunk(b"payload-one", meta=b"meta1", chunk_id=H(1))
+    obj2 = bytearray(fchunk(b"d2", meta=b"m2", chunk_id=H(2)))
+    obj2[0] ^= 0xFF  # break the magic of the second object's header
+    reader = PackReader(pack_contents=obj1 + bytes(obj2))
+    with pytest.raises(IntegrityError):
+        list(reader.iter_headers())
+
+
+def test_pack_reader_raises_on_bad_magic_through_store(tmp_path):
+    obj = bytearray(fchunk(b"FIRST", chunk_id=H(47)))
+    obj[0] ^= 0xFF
+    pack_id = H(44)
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        repository.store_store("packs/" + bin_to_hex(pack_id), bytes(obj))
+        reader = PackReader(repository.store, pack_id)
+        with pytest.raises(IntegrityError):
+            list(reader.iter_headers())
+
+
+def test_pack_reader_raises_on_object_past_end_of_pack():
+    # a valid header whose declared sizes overrun the pack: the object cannot be there.
+    obj = fchunk(b"data", meta=b"meta", chunk_id=H(5))
+    pack = obj[:-1]  # drop a byte, so the header's data_size no longer fits
+    reader = PackReader(pack_contents=pack)
+    with pytest.raises(IntegrityError):
+        list(reader.iter_headers())
+
+
+def test_pack_reader_raises_on_object_past_end_of_pack_through_store(tmp_path):
+    obj = fchunk(b"FIRST", chunk_id=H(49))
+    pack_id = H(45)
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        repository.store_store("packs/" + bin_to_hex(pack_id), obj[:-1])
+        reader = PackReader(repository.store, pack_id)
+        with pytest.raises(IntegrityError):
+            list(reader.iter_headers())
+
+
+def test_pack_reader_size(tmp_path):
+    obj = fchunk(b"data", meta=b"meta", chunk_id=H(6))
+    assert PackReader(pack_contents=obj).size() == len(obj)
+    pack_id = H(46)
+    with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
+        repository.store_store("packs/" + bin_to_hex(pack_id), obj)
+        assert PackReader(repository.store, pack_id).size() == len(obj)
+
+
 def test_pack_reader_in_memory_read_returns_view():
     # read() over an in-memory pack returns a memoryview into pack_contents.
     obj1 = fchunk(b"payload-one", meta=b"meta1", chunk_id=H(1))

From 3a46540d4791f3456e67eb29cd117afc74e23366 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 13 Aug 2026 01:29:16 +0530
Subject: [PATCH 106/144] check: guard cross-check on interrupt, make
 missing-pack message actionable (#9898)

---
 src/borg/repository.py                | 19 ++++++++++++-------
 src/borg/testsuite/repository_test.py | 15 +++++----------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index 35b4c50baa..76ec63f426 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1114,16 +1114,19 @@ def recorded_ts(info):
             present_pack_ids = {hex_to_bin(info.name) for info in pack_infos}
             tracker.prune(present_pack_ids)
             pack_pi.finish()
-            # cross-check the chunk index against packs/ (refs #9898). skipped for an invalid index:
-            # its fragment set may be incomplete, and build_chunkindex_from_repo would delete fragments
-            # and rebuild (a write), which a read-only check must not do.
-            if not index_invalid:
+            # cross-check the chunk index against packs/ (refs #9898). skip it when the index is
+            # invalid (loading it rebuilds from packs, which writes, and a check must not write) or
+            # after Ctrl-C (sig_int), where building the whole index would negate the interrupt.
+            if not index_invalid and not sig_int:
                 chunks = build_chunkindex_from_repo(self)
                 try:
                     referenced_pack_ids = {
                         entry.pack_id
                         for _, entry in chunks.iteritems()
-                        if not (entry.flags & ChunkIndex.F_PENDING)  # pending: pack_id not resolved yet
+                        # F_PENDING means the chunk's pack location is unresolved, so pack_id is a
+                        # placeholder, not a real pack. Fragments should not hold pending entries;
+                        # skip them defensively.
+                        if not (entry.flags & ChunkIndex.F_PENDING)
                     }
                 finally:
                     chunks.clear()
@@ -1144,8 +1147,6 @@ def recorded_ts(info):
             f"Checked {index_files} index files ({index_errors} errors) "
             f"and {pack_files} packs ({pack_errors} errors)."
         )
-        if missing_pack_ids:
-            summary += f" {len(missing_pack_ids)} pack(s) referenced by the index are missing."
         if pack_skipped:
             summary += f" Reused {pack_skipped} recent pack check result(s)."
         logger.info(summary)
@@ -1154,6 +1155,10 @@ def recorded_ts(info):
             logger.error(f"{len(missing_pack_ids)} pack(s) referenced by the index are missing:")
             for pack_id in missing_pack_ids:
                 logger.error(f"Missing pack: {bin_to_hex(pack_id)}")
+            logger.error(
+                "The chunks stored in these packs are lost. Repairing the index (dropping the "
+                "stale references) is tracked in https://github.com/borgbackup/borg/issues/8572."
+            )
         # corrupt_ids() is every pack recorded corrupt, including from earlier runs. with a corrupt
         # index the packs were not scanned, so report nothing.
         corrupt_ids = tracker.corrupt_ids() if index_errors == 0 else []
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 8c2c6d411e..1fc9a5b99b 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -9,6 +9,7 @@
 import pytest
 from borghash import HashTableNT
 
+from ..cache import write_chunkindex_invalid
 from ..constants import MAX_CLOCK_SKEW
 from ..helpers import IntegrityError, Location, bin_to_hex
 from ..hashindex import ChunkIndex
@@ -1024,12 +1025,11 @@ def boom(*args, **kwargs):
         assert H(1) not in repository._chunks
 
 
-def _serialized_chunkindex(chunks=None):
-    # Serialize a ChunkIndex to bytes, as stored under index/<sha256(content)>. check() parses index
-    # fragments, so a fragment must be a real ChunkIndex serialization.
-    chunks = chunks if chunks is not None else ChunkIndex()
+def _serialized_chunkindex():
+    # Serialize an empty ChunkIndex to bytes, as stored under index/<sha256(content)>. check() parses
+    # index fragments, so a fragment must be a real ChunkIndex serialization.
     with io.BytesIO() as f:
-        chunks.write(f)
+        ChunkIndex().write(f)
         return f.getvalue()
 
 
@@ -1083,9 +1083,6 @@ def test_check_reports_invalid_pack_name(tmp_path, caplog):
 def test_check_warns_on_invalid_chunk_index(tmp_path, caplog):
     # check warns about an invalid chunk index but does not fail, since the index is not part of
     # the repository's object integrity.
-    import logging
-    from ..cache import write_chunkindex_invalid
-
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         write_chunkindex_invalid(repository)
         with caplog.at_level(logging.WARNING):
@@ -1123,8 +1120,6 @@ def test_check_detects_missing_pack_referenced_by_index(tmp_path, caplog):
 def test_check_missing_pack_detection_skipped_when_index_invalid(tmp_path, caplog):
     # an invalid index is rebuilt from the packs on next use, so check does not report missing packs
     # from it (refs #9898); it only warns about the invalid index.
-    from ..cache import write_chunkindex_invalid
-
     location = os.fspath(tmp_path / "repo")
     with Repository(location, exclusive=True, create=True) as repository:
         for x in range(3):

From 578c93a224a59d8f6e6025f5335650e235e2d964 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 13 Aug 2026 01:53:44 +0530
Subject: [PATCH 107/144] check: cross-check index from fragments only, skip on
 partial runs (#9898)

Read the chunk index from its fragments only for the missing-pack cross-check;
skip it (still passing) when the index cannot be read that way, and on partial
(--max-duration) runs. Guard ChunkIndex.read against hash-valid-but-unreadable
fragments.
---
 src/borg/cache.py                     | 20 +++++++--
 src/borg/repository.py                | 63 +++++++++++++++------------
 src/borg/testsuite/repository_test.py | 36 +++++++++++++++
 3 files changed, 89 insertions(+), 30 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index 0717f87c72..d2d86b728e 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -778,8 +778,13 @@ def read_chunkindex_from_repo(repository, hash):
     else:
         if hashlib.sha256(chunks_data).digest() == hex_to_bin(hash):
             logger.debug(f"{index_name} is valid.")
-            with io.BytesIO(chunks_data) as f:
-                chunks = ChunkIndex.read(f)
+            try:
+                with io.BytesIO(chunks_data) as f:
+                    chunks = ChunkIndex.read(f)
+            except Exception:
+                # name matches the content hash, but the content does not deserialize into a ChunkIndex.
+                logger.warning(f"{index_name} has a valid name but unreadable content; treating it as invalid.")
+                return None
             return chunks
         else:
             logger.debug(f"{index_name} is invalid.")
@@ -839,8 +844,11 @@ def repack_chunkindex(repository):
 
 
 def build_chunkindex_from_repo(
-    repository, *, slow_rebuild=False, write_immediately=False, init_flags=ChunkIndex.F_USED
+    repository, *, slow_rebuild=False, fragments_only=False, write_immediately=False, init_flags=ChunkIndex.F_USED
 ):
+    # fragments_only: build the index from the index/ fragments only, returning None if they cannot be
+    # read completely, and never write to the repo.
+    assert not (slow_rebuild and fragments_only)
     # first, try to build a fresh, mostly complete chunk index from centrally stored index fragments:
     if not slow_rebuild:
         # a concurrent repack_chunkindex (another client, shared lock) deletes the small fragments it
@@ -852,6 +860,8 @@ def build_chunkindex_from_repo(
         # set (e.g. a persistently unreadable fragment), fall through to the slow rebuild instead.
         for _ in range(CHUNKINDEX_MERGE_ATTEMPTS):
             if chunkindex_is_invalid(repository):
+                if fragments_only:
+                    return None
                 # leftover fragments may be incomplete or stale. Finish the interrupted deletion
                 # (best-effort; a read-only client rebuilds in memory only), then rebuild from packs.
                 logger.warning("chunk index is invalid (interrupted operation), rebuilding it.")
@@ -862,6 +872,8 @@ def build_chunkindex_from_repo(
                 break
             hashes = list_chunkindex_hashes(repository)
             if not hashes:  # no chunk index fragments available
+                if fragments_only:
+                    return None
                 break
             chunks = ChunkIndex()  # we'll merge all fragments into this
             complete = True
@@ -887,6 +899,8 @@ def build_chunkindex_from_repo(
                 return chunks
             chunks.clear()  # free the partial merge before retrying
         else:
+            if fragments_only:
+                return None
             logger.warning("could not read a complete set of chunk index fragments, rebuilding the index.")
     # if we didn't get anything from the index fragments, compute the ChunkIndex the slow way:
     logger.debug("rebuilding the chunk index from the repo the slow way...")
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 76ec63f426..03a1f6ab37 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -987,9 +987,12 @@ def check(self, repair=False, max_duration=0, max_age=0):
         compact, or salvaged and dropped by repair; refs #8572); prune() does this from packs/.
 
         It also reports missing packs (refs #9898): pack ids the chunk index references but that are
-        absent from packs/. The chunk index is loaded and its referenced pack ids are compared with the
-        packs present in the store. A corrupt or invalid index is rebuilt from the packs on next use, so
-        its references are not checked here.
+        absent from packs/. The index is read from its fragments only and its referenced pack ids are
+        compared with the packs present in the store. This cross-check is skipped, and the check still
+        passes, when the index cannot be read from its fragments (an invalid index is regenerated from
+        the packs on next use, so it can never reference a missing pack; a pack that is truly gone then
+        surfaces as missing chunks in the archives check). It is also skipped on a partial run
+        (max_duration), which covers packs incrementally; the cross-check runs on the full check instead.
 
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
         max_age, accepting a future timestamp up to MAX_CLOCK_SKEW (clock skew). Results are recorded
@@ -1114,31 +1117,37 @@ def recorded_ts(info):
             present_pack_ids = {hex_to_bin(info.name) for info in pack_infos}
             tracker.prune(present_pack_ids)
             pack_pi.finish()
-            # cross-check the chunk index against packs/ (refs #9898). skip it when the index is
-            # invalid (loading it rebuilds from packs, which writes, and a check must not write) or
-            # after Ctrl-C (sig_int), where building the whole index would negate the interrupt.
-            if not index_invalid and not sig_int:
-                chunks = build_chunkindex_from_repo(self)
-                try:
-                    referenced_pack_ids = {
-                        entry.pack_id
-                        for _, entry in chunks.iteritems()
-                        # F_PENDING means the chunk's pack location is unresolved, so pack_id is a
-                        # placeholder, not a real pack. Fragments should not hold pending entries;
-                        # skip them defensively.
-                        if not (entry.flags & ChunkIndex.F_PENDING)
-                    }
-                finally:
-                    chunks.clear()
-                # index entries pointing to a pack absent from packs/: data loss.
-                missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
-                # packs no index entry references: leftovers of an interrupted operation, not an error.
-                orphan_pack_ids = sorted(present_pack_ids - referenced_pack_ids)
-                if orphan_pack_ids:
-                    logger.info(
-                        f"{len(orphan_pack_ids)} pack(s) in packs/ are not referenced by the index "
-                        f"(leftovers from an interrupted operation)."
+            # cross-check the chunk index against packs/ to find referenced-but-absent packs (refs #9898).
+            # read the index from its fragments only; if that fails, skip the cross-check (rebuilding from
+            # packs is too slow here, and a check must not write). also skip after Ctrl-C (sig_int) and on
+            # a partial run (--max-duration), which the full check's cross-check covers.
+            if not index_invalid and not sig_int and not partial:
+                chunks = build_chunkindex_from_repo(self, fragments_only=True)
+                if chunks is None:
+                    logger.warning(
+                        "Cannot cross-check packs against the chunk index: the index could not be loaded "
+                        "from its fragments; skipping missing-pack detection."
                     )
+                else:
+                    try:
+                        referenced_pack_ids = {
+                            entry.pack_id
+                            for _, entry in chunks.iteritems()
+                            # F_PENDING marks a chunk whose pack location is unresolved, so its pack_id
+                            # is a placeholder rather than a real pack; skip such entries.
+                            if not (entry.flags & ChunkIndex.F_PENDING)
+                        }
+                    finally:
+                        chunks.clear()
+                    # index entries pointing to a pack absent from packs/: data loss.
+                    missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
+                    # packs no index entry references: leftovers of an interrupted operation, not an error.
+                    orphan_pack_ids = sorted(present_pack_ids - referenced_pack_ids)
+                    if orphan_pack_ids:
+                        logger.info(
+                            f"{len(orphan_pack_ids)} pack(s) in packs/ are not referenced by the index "
+                            f"(leftovers from an interrupted operation)."
+                        )
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 1fc9a5b99b..09199cdda8 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1152,6 +1152,42 @@ def test_check_reports_orphan_pack_not_referenced_by_index(tmp_path, caplog):
         assert "not referenced by the index" in caplog.text
 
 
+def test_check_missing_pack_detection_skipped_when_index_unreadable(tmp_path, caplog):
+    # an index/ fragment whose name matches its content hash but whose content does not deserialize
+    # into a ChunkIndex makes the fragment set unreadable; check skips the cross-check (and still
+    # passes) instead of crashing or rebuilding from the packs (refs #9898).
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()
+    with Repository(location, exclusive=True) as repository:
+        pack_id = repository.chunks[H(0)].pack_id
+        repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone, index entry kept
+        content = b"not a serialized chunk index"
+        repository.store_store("index/" + bin_to_hex(sha256(content).digest()), content)
+        with caplog.at_level(logging.WARNING):
+            assert repository.check(repair=False) is True
+        assert "Missing pack" not in caplog.text
+        assert "Cannot cross-check packs against the chunk index" in caplog.text
+
+
+def test_check_partial_skips_missing_pack_cross_check(tmp_path, caplog):
+    # a partial check (max_duration) does not cross-check the index against packs/; missing-pack
+    # detection runs on the full check instead (refs #9898).
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()
+    with Repository(location, exclusive=True) as repository:
+        pack_id = repository.chunks[H(0)].pack_id
+        repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone, index entry kept
+        with caplog.at_level(logging.ERROR):
+            assert repository.check(repair=False, max_duration=3600) is True
+        assert "Missing pack" not in caplog.text
+
+
 def test_check_checked_packs_roundtrip(tmp_path):
     # the set survives a store/load round-trip; a rotted blob loads as empty.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:

From 2b2bff3f5db1193dd54af95346da5c10c540a57f Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 13 Aug 2026 09:25:25 +0530
Subject: [PATCH 108/144] check: cross-check index before pack loop so partial
 runs detect missing packs, reuse it as .chunks (#9898)

---
 src/borg/repository.py                | 70 +++++++++++++--------------
 src/borg/testsuite/repository_test.py | 10 ++--
 2 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index 03a1f6ab37..98572cddc2 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -988,11 +988,11 @@ def check(self, repair=False, max_duration=0, max_age=0):
 
         It also reports missing packs (refs #9898): pack ids the chunk index references but that are
         absent from packs/. The index is read from its fragments only and its referenced pack ids are
-        compared with the packs present in the store. This cross-check is skipped, and the check still
+        compared with the packs present in the store. This cross-check runs before the pack loop, so
+        max_duration bounds it and it runs on partial runs too. It is skipped, and the check still
         passes, when the index cannot be read from its fragments (an invalid index is regenerated from
         the packs on next use, so it can never reference a missing pack; a pack that is truly gone then
-        surfaces as missing chunks in the archives check). It is also skipped on a partial run
-        (max_duration), which covers packs incrementally; the cross-check runs on the full check instead.
+        surfaces as missing chunks in the archives check).
 
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
         max_age, accepting a future timestamp up to MAX_CLOCK_SKEW (clock skew). Results are recorded
@@ -1070,6 +1070,38 @@ def store_list(namespace):
                     logger.error(f"Store object packs/{info.name} has an invalid name.")
                     pack_errors += 1
             pack_infos = valid_pack_infos
+            present_pack_ids = {hex_to_bin(info.name) for info in pack_infos}
+            # cross-check the chunk index against packs/ to find referenced-but-absent packs (refs #9898).
+            # run before the pack loop so max_duration bounds it and partial checks cover it too. read
+            # the index from its fragments only: a full rebuild reads every pack (too slow) and writes
+            # to the repo (a check must not).
+            if not index_invalid and not sig_int:
+                chunks = build_chunkindex_from_repo(self, fragments_only=True)
+                if chunks is None:
+                    logger.warning(
+                        "Cannot cross-check packs against the chunk index: the index could not be loaded "
+                        "from its fragments; skipping missing-pack detection."
+                    )
+                else:
+                    referenced_pack_ids = {
+                        entry.pack_id
+                        for _, entry in chunks.iteritems()
+                        # F_PENDING marks a chunk whose pack location is unresolved, so its pack_id
+                        # is a placeholder rather than a real pack; skip such entries.
+                        if not (entry.flags & ChunkIndex.F_PENDING)
+                    }
+                    # set the session chunk index (.chunks) so later reads reuse it; clear_new() has
+                    # run, so close() does not write it back.
+                    self.chunks = chunks
+                    # index entries pointing to a pack absent from packs/: data loss.
+                    missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
+                    # packs no index entry references: leftovers of an interrupted operation, not an error.
+                    orphan_pack_ids = sorted(present_pack_ids - referenced_pack_ids)
+                    if orphan_pack_ids:
+                        logger.info(
+                            f"{len(orphan_pack_ids)} pack(s) in packs/ are not referenced by the index "
+                            f"(leftovers from an interrupted operation)."
+                        )
             if partial:
                 # a partial check stops after max_duration; verify the least-recently-checked packs
                 # first so repeated runs cover every pack. sort by recorded check time, unrecorded
@@ -1114,40 +1146,8 @@ def recorded_ts(info):
                 if pack_infos:
                     pack_pi.show(current=len(pack_infos))  # finish at 100%
                 logger.info("Finished checking packs.")
-            present_pack_ids = {hex_to_bin(info.name) for info in pack_infos}
             tracker.prune(present_pack_ids)
             pack_pi.finish()
-            # cross-check the chunk index against packs/ to find referenced-but-absent packs (refs #9898).
-            # read the index from its fragments only; if that fails, skip the cross-check (rebuilding from
-            # packs is too slow here, and a check must not write). also skip after Ctrl-C (sig_int) and on
-            # a partial run (--max-duration), which the full check's cross-check covers.
-            if not index_invalid and not sig_int and not partial:
-                chunks = build_chunkindex_from_repo(self, fragments_only=True)
-                if chunks is None:
-                    logger.warning(
-                        "Cannot cross-check packs against the chunk index: the index could not be loaded "
-                        "from its fragments; skipping missing-pack detection."
-                    )
-                else:
-                    try:
-                        referenced_pack_ids = {
-                            entry.pack_id
-                            for _, entry in chunks.iteritems()
-                            # F_PENDING marks a chunk whose pack location is unresolved, so its pack_id
-                            # is a placeholder rather than a real pack; skip such entries.
-                            if not (entry.flags & ChunkIndex.F_PENDING)
-                        }
-                    finally:
-                        chunks.clear()
-                    # index entries pointing to a pack absent from packs/: data loss.
-                    missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
-                    # packs no index entry references: leftovers of an interrupted operation, not an error.
-                    orphan_pack_ids = sorted(present_pack_ids - referenced_pack_ids)
-                    if orphan_pack_ids:
-                        logger.info(
-                            f"{len(orphan_pack_ids)} pack(s) in packs/ are not referenced by the index "
-                            f"(leftovers from an interrupted operation)."
-                        )
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index 09199cdda8..e58431e2f6 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1172,9 +1172,9 @@ def test_check_missing_pack_detection_skipped_when_index_unreadable(tmp_path, ca
         assert "Cannot cross-check packs against the chunk index" in caplog.text
 
 
-def test_check_partial_skips_missing_pack_cross_check(tmp_path, caplog):
-    # a partial check (max_duration) does not cross-check the index against packs/; missing-pack
-    # detection runs on the full check instead (refs #9898).
+def test_check_partial_still_detects_missing_pack(tmp_path, caplog):
+    # a partial check (max_duration) cross-checks the index against packs/ before the pack loop, so
+    # it detects a missing pack and fails just like a full check (refs #9898).
     location = os.fspath(tmp_path / "repo")
     with Repository(location, exclusive=True, create=True) as repository:
         for x in range(3):
@@ -1184,8 +1184,8 @@ def test_check_partial_skips_missing_pack_cross_check(tmp_path, caplog):
         pack_id = repository.chunks[H(0)].pack_id
         repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone, index entry kept
         with caplog.at_level(logging.ERROR):
-            assert repository.check(repair=False, max_duration=3600) is True
-        assert "Missing pack" not in caplog.text
+            assert repository.check(repair=False, max_duration=3600) is False
+        assert f"Missing pack: {bin_to_hex(pack_id)}" in caplog.text
 
 
 def test_check_checked_packs_roundtrip(tmp_path):

From 4d9fb17e26781482422c0c84575429b447ae2790 Mon Sep 17 00:00:00 2001
From: mrityunjay <mrityunjay@even.in>
Date: Thu, 13 Aug 2026 04:34:41 +0000
Subject: [PATCH 109/144] check: raise CorruptChunkIndexFragment for unreadable
 fragments so a corrupt fragment rebuilds instead of retrying, add
 fragments_only write guard

compact/repo-compress/check: align stale-pack wording to refs #8572, fix singular grammar; list orphan pack ids at debug (#9898)
---
 src/borg/archiver/compact_cmd.py       | 19 ++++++++++----
 src/borg/archiver/repo_compress_cmd.py | 13 +++++++---
 src/borg/cache.py                      | 36 +++++++++++++++++++++-----
 src/borg/repository.py                 |  9 +++----
 src/borg/testsuite/cache_test.py       | 30 +++++++++++++++++++++
 5 files changed, 87 insertions(+), 20 deletions(-)

diff --git a/src/borg/archiver/compact_cmd.py b/src/borg/archiver/compact_cmd.py
index 6fc9ed3008..d62815f2b1 100644
--- a/src/borg/archiver/compact_cmd.py
+++ b/src/borg/archiver/compact_cmd.py
@@ -287,20 +287,29 @@ def compact_packs(self):
                     pack_used[pid] += entry.obj_size
 
         if stale_ids:
-            # keep these entries: they may be an archive's only pointer to a chunk. dropping them is
-            # "borg check --repair"'s call.
-            logger.warning(f'{len(stale_ids)} index entries reference missing pack files; run "borg check --repair".')
+            # keep these entries: they may be an archive's only pointer to a chunk. dropping them
+            # (repairing the index) is not implemented yet, refs #8572.
+            n = len(stale_ids)
+            if n == 1:
+                subject = "1 index entry references a missing pack file"
+            else:
+                subject = f"{n} index entries reference missing pack files"
+            logger.warning(
+                f"{subject}. Repairing the index (dropping the stale references) is tracked in "
+                "https://github.com/borgbackup/borg/issues/8572."
+            )
             if stale_used:
                 logger.error(f"{stale_used} of them are still in use: repository data is missing!")
                 set_ec(EXIT_ERROR)
 
         # bytes no index entry covers. compact_pack reclaims the redundant duplicates among them while
-        # rewriting a pack; the rest is left for "borg check --repair".
+        # rewriting a pack; reclaiming the rest is tracked in #8572.
         unindexed = sum(total - pack_indexed[pid] for pid, total in pack_total.items() if total > pack_indexed[pid])
         if unindexed:
             logger.info(
                 f"{format_file_size(unindexed)} in pack files is not covered by the index; "
-                'redundant copies are reclaimed on pack rewrite, the rest by "borg check --repair".'
+                "redundant copies are reclaimed on pack rewrite, reclaiming the rest is tracked in "
+                "https://github.com/borgbackup/borg/issues/8572."
             )
 
         # decide each pack's fate. a pack's reclaimable bytes are its indexed-but-unused bytes; the
diff --git a/src/borg/archiver/repo_compress_cmd.py b/src/borg/archiver/repo_compress_cmd.py
index 1a5089a617..b127007418 100644
--- a/src/borg/archiver/repo_compress_cmd.py
+++ b/src/borg/archiver/repo_compress_cmd.py
@@ -93,10 +93,17 @@ def recompress(self):
 
         stale_packs = set(per_pack) - {pack_id for pack_id, _ in packs}
         if stale_packs:
-            # keep these entries: they may be an archive's only pointer to a chunk. dropping them is
-            # "borg check --repair"'s call.
+            # keep these entries: they may be an archive's only pointer to a chunk. dropping them
+            # (repairing the index) is not implemented yet, refs #8572.
             stale = sum(len(per_pack[pack_id]) for pack_id in stale_packs)
-            logger.warning(f'{stale} index entries reference missing pack files; run "borg check --repair".')
+            if stale == 1:
+                subject = "1 index entry references a missing pack file"
+            else:
+                subject = f"{stale} index entries reference missing pack files"
+            logger.warning(
+                f"{subject}. Repairing the index (dropping the stale references) is tracked in "
+                "https://github.com/borgbackup/borg/issues/8572."
+            )
 
         pi = ProgressIndicatorPercent(
             total=len(packs), msg="Recompressing %3.1f%%", step=0.1, msgid="repo_compress.recompress"
diff --git a/src/borg/cache.py b/src/borg/cache.py
index d2d86b728e..e49e3533be 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -768,6 +768,10 @@ def gen_batches():
     return new_hashes
 
 
+class CorruptChunkIndexFragment(Exception):
+    """A chunk index fragment's name matches its content hash, but the content does not deserialize."""
+
+
 def read_chunkindex_from_repo(repository, hash):
     index_name = f"index/{hash}"
     logger.debug(f"trying to load {index_name} from the repo...")
@@ -781,10 +785,10 @@ def read_chunkindex_from_repo(repository, hash):
             try:
                 with io.BytesIO(chunks_data) as f:
                     chunks = ChunkIndex.read(f)
-            except Exception:
-                # name matches the content hash, but the content does not deserialize into a ChunkIndex.
-                logger.warning(f"{index_name} has a valid name but unreadable content; treating it as invalid.")
-                return None
+            except (ValueError, KeyError, struct.error) as err:
+                # the name matches the content hash, so the bytes are intact but do not deserialize
+                # into a ChunkIndex: the fragment is corrupt.
+                raise CorruptChunkIndexFragment(index_name) from err
             return chunks
         else:
             logger.debug(f"{index_name} is invalid.")
@@ -822,7 +826,11 @@ def repack_chunkindex(repository):
     merged = ChunkIndex()
     merged_hashes = []
     for name, _ in small:
-        fragment = read_chunkindex_from_repo(repository, name)
+        try:
+            fragment = read_chunkindex_from_repo(repository, name)
+        except CorruptChunkIndexFragment:
+            # a corrupt fragment cannot be merged; leave it in place and skip it.
+            continue
         if fragment is None:
             # gone or invalid (e.g. deleted by another client); just don't merge it.
             continue
@@ -849,6 +857,7 @@ def build_chunkindex_from_repo(
     # fragments_only: build the index from the index/ fragments only, returning None if they cannot be
     # read completely, and never write to the repo.
     assert not (slow_rebuild and fragments_only)
+    assert not (fragments_only and write_immediately)  # fragments_only never writes to the repo
     # first, try to build a fresh, mostly complete chunk index from centrally stored index fragments:
     if not slow_rebuild:
         # a concurrent repack_chunkindex (another client, shared lock) deletes the small fragments it
@@ -877,16 +886,29 @@ def build_chunkindex_from_repo(
                 break
             chunks = ChunkIndex()  # we'll merge all fragments into this
             complete = True
+            corrupt_fragment = None
             for hash in hashes:
-                chunks_to_merge = read_chunkindex_from_repo(repository, hash)
+                try:
+                    chunks_to_merge = read_chunkindex_from_repo(repository, hash)
+                except CorruptChunkIndexFragment as err:
+                    corrupt_fragment = err
+                    break
                 if chunks_to_merge is None:
-                    logger.debug(f"chunk index fragment {hash} vanished or is invalid, restarting the merge...")
+                    logger.debug(f"chunk index fragment {hash} vanished, restarting the merge...")
                     complete = False
                     break
                 logger.debug(f"chunk index fragment {hash} gets merged...")
                 for k, v in chunks_to_merge.items():
                     chunks[k] = v
                 chunks_to_merge.clear()
+            if corrupt_fragment is not None:
+                # retrying would re-read the same corrupt fragment; rebuild the whole index from
+                # the packs instead (or return None in fragments_only mode).
+                chunks.clear()
+                if fragments_only:
+                    return None
+                logger.warning(f"{corrupt_fragment} is corrupt, rebuilding the chunk index from the packs.")
+                break
             if complete:
                 if len(hashes) > 1 and write_immediately:
                     # consolidate small fragments on the repo so they don't pile up. this is a
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 98572cddc2..7d2b518512 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1095,13 +1095,12 @@ def store_list(namespace):
                     self.chunks = chunks
                     # index entries pointing to a pack absent from packs/: data loss.
                     missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
-                    # packs no index entry references: leftovers of an interrupted operation, not an error.
+                    # packs no index entry references: not an error, so info + ids at debug only.
                     orphan_pack_ids = sorted(present_pack_ids - referenced_pack_ids)
                     if orphan_pack_ids:
-                        logger.info(
-                            f"{len(orphan_pack_ids)} pack(s) in packs/ are not referenced by the index "
-                            f"(leftovers from an interrupted operation)."
-                        )
+                        logger.info(f"{len(orphan_pack_ids)} pack(s) are not referenced by the index.")
+                        for pack_id in orphan_pack_ids:
+                            logger.debug(f"Orphan pack: {bin_to_hex(pack_id)}")
             if partial:
                 # a partial check stops after max_duration; verify the least-recently-checked packs
                 # first so repeated runs cover every pack. sort by recorded check time, unrecorded
diff --git a/src/borg/testsuite/cache_test.py b/src/borg/testsuite/cache_test.py
index b82e6b45bc..872499c197 100644
--- a/src/borg/testsuite/cache_test.py
+++ b/src/borg/testsuite/cache_test.py
@@ -12,6 +12,7 @@
 from ..cache import (
     AdHocWithFilesCache,
     ChunksMixin,
+    CorruptChunkIndexFragment,
     FileCacheEntry,
     build_chunkindex_from_repo,
     chunkindex_is_invalid,
@@ -146,6 +147,35 @@ def test_read_chunkindex_from_repo_missing(tmp_path):
         assert result is None
 
 
+def test_read_chunkindex_from_repo_corrupt(tmp_path):
+    """A fragment whose name matches its content hash but does not deserialize raises."""
+    repository_location = os.fspath(tmp_path / "repository")
+    with Repository(repository_location, exclusive=True, create=True) as repository:
+        content = b"not a serialized chunk index"
+        name = hashlib.sha256(content).hexdigest()  # valid name, so the name check passes
+        repository.store_store(f"index/{name}", content)
+        with pytest.raises(CorruptChunkIndexFragment):
+            read_chunkindex_from_repo(repository, name)
+
+
+def test_build_chunkindex_rebuilds_on_corrupt_fragment(tmp_path):
+    """A corrupt fragment (valid name, unreadable content) triggers a rebuild from the packs, no retry."""
+    repository_location = os.fspath(tmp_path / "repository")
+    with Repository(repository_location, exclusive=True, create=True) as repository:
+        ci = ChunkIndex()
+        ci[H(1)] = ChunkIndexEntry(ChunkIndex.F_NEW, 0, H(1), 0, 4)
+        write_chunkindex_to_repo(repository, ci, incremental=False, force_write=True)
+        content = b"not a serialized chunk index"
+        name = hashlib.sha256(content).hexdigest()
+        repository.store_store(f"index/{name}", content)
+        chunks = build_chunkindex_from_repo(repository)
+        # the rebuild reads the (empty) packs namespace, so H(1) from the corrupt fragment is absent
+        assert H(1) not in chunks
+        assert len(chunks) == 0
+        # fragments_only returns None here rather than rebuilding from the packs
+        assert build_chunkindex_from_repo(repository, fragments_only=True) is None
+
+
 def test_build_chunkindex_retries_on_vanished_fragment(tmp_path):
     """build_chunkindex_from_repo must re-list and retry when a fragment vanishes mid-merge.
 

From f500b0f7246e9dfedd3f2cc2677613edc462940e Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 13 Aug 2026 14:32:26 +0530
Subject: [PATCH 110/144] compact/repo-compress: single stale-pack message form
 for any count (#9898)

---
 src/borg/archiver/compact_cmd.py       | 8 ++------
 src/borg/archiver/repo_compress_cmd.py | 8 ++------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/borg/archiver/compact_cmd.py b/src/borg/archiver/compact_cmd.py
index d62815f2b1..00e78680ec 100644
--- a/src/borg/archiver/compact_cmd.py
+++ b/src/borg/archiver/compact_cmd.py
@@ -290,13 +290,9 @@ def compact_packs(self):
             # keep these entries: they may be an archive's only pointer to a chunk. dropping them
             # (repairing the index) is not implemented yet, refs #8572.
             n = len(stale_ids)
-            if n == 1:
-                subject = "1 index entry references a missing pack file"
-            else:
-                subject = f"{n} index entries reference missing pack files"
             logger.warning(
-                f"{subject}. Repairing the index (dropping the stale references) is tracked in "
-                "https://github.com/borgbackup/borg/issues/8572."
+                f"index entries referencing a missing pack file: {n}. Repairing the index "
+                "(dropping the stale references) is tracked in https://github.com/borgbackup/borg/issues/8572."
             )
             if stale_used:
                 logger.error(f"{stale_used} of them are still in use: repository data is missing!")
diff --git a/src/borg/archiver/repo_compress_cmd.py b/src/borg/archiver/repo_compress_cmd.py
index b127007418..273abc787e 100644
--- a/src/borg/archiver/repo_compress_cmd.py
+++ b/src/borg/archiver/repo_compress_cmd.py
@@ -96,13 +96,9 @@ def recompress(self):
             # keep these entries: they may be an archive's only pointer to a chunk. dropping them
             # (repairing the index) is not implemented yet, refs #8572.
             stale = sum(len(per_pack[pack_id]) for pack_id in stale_packs)
-            if stale == 1:
-                subject = "1 index entry references a missing pack file"
-            else:
-                subject = f"{stale} index entries reference missing pack files"
             logger.warning(
-                f"{subject}. Repairing the index (dropping the stale references) is tracked in "
-                "https://github.com/borgbackup/borg/issues/8572."
+                f"index entries referencing a missing pack file: {stale}. Repairing the index "
+                "(dropping the stale references) is tracked in https://github.com/borgbackup/borg/issues/8572."
             )
 
         pi = ProgressIndicatorPercent(

From 5b9da3bc0b8a46b27e8590f7bfd4f8d5683cf8d4 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 13 Aug 2026 11:49:21 +0200
Subject: [PATCH 111/144] create: add --read-special-timeout option, fixes
 #5422

Reading from a fifo (or char device) with --read-special could block
borg forever: opening a fifo read-only blocks until a writer connects,
and reads block until data arrives - a broken producer thus hangs an
unattended backup indefinitely.

Now, when no data arrives for --read-special-timeout SECONDS (default:
1800, i.e. 30 minutes; this includes waiting for a fifo's writer to
connect), borg gives up on that file: fifos and char devices are opened
with O_NONBLOCK and all content reads go through the new
SpecialFileReader, which waits for data via select() and raises
OSError(ETIMEDOUT) when the timeout expires. That maps to the new
BackupTimeoutError (rc 111), so the file is skipped with an error (and
not retried) while the backup continues with the remaining files.
--read-special-timeout=0 means: wait forever (previous behavior).

Note: with a timeout active (the default), a writer connecting but
closing without sending anything is reported as a timeout instead of
being stored as a file with empty content.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/internals/frontends.rst                  |   2 +
 docs/man/borg-create.1                        |   9 +-
 docs/usage/create.rst.inc                     | 197 +++++++++---------
 docs/usage/notes.rst                          |  12 ++
 src/borg/archive.py                           |  29 ++-
 src/borg/archiver/create_cmd.py               |  27 ++-
 src/borg/constants.py                         |   5 +
 src/borg/helpers/__init__.py                  |   5 +-
 src/borg/helpers/errors.py                    |   8 +
 src/borg/helpers/fs.py                        |  64 ++++++
 src/borg/helpers/parseformat.py               |  11 +
 .../testsuite/archiver/create_cmd_test.py     | 146 ++++++++++++-
 src/borg/testsuite/helpers/fs_test.py         | 140 ++++++++++++-
 .../testsuite/helpers/parseformat_test.py     |  12 ++
 14 files changed, 552 insertions(+), 115 deletions(-)

diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst
index 9b9bbe8fb3..6af8114b2d 100644
--- a/docs/internals/frontends.rst
+++ b/docs/internals/frontends.rst
@@ -735,6 +735,8 @@ Warnings
         {}: {}
     BackupFileNotFoundError rc: 107
         {}: {}
+    BackupTimeoutError rc: 111
+        {}: {}
 
 Operations
     - cache.begin_transaction
diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1
index e302b853f1..a5fc9ece74 100644
--- a/docs/man/borg-create.1
+++ b/docs/man/borg-create.1
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "borg-create" "1" "2026-07-21" "" "borg backup tool"
+.TH "borg-create" "1" "2026-08-13" "" "borg backup tool"
 .SH Name
 borg-create \- Creates a new archive.
 .SH SYNOPSIS
@@ -282,6 +282,9 @@ specify how to detect if a file has changed during backup (ctime, mtime, disable
 .TP
 .B  \-\-read\-special
 open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.
+.TP
+.BI \-\-read\-special\-timeout \ SECONDS
+when reading from FIFOs or character devices (see \-\-read\-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO\(aqs writer to connect). Give 0 to wait forever. default: 1800 seconds.
 .UNINDENT
 .SS Archive options
 .INDENT 0.0
@@ -293,7 +296,7 @@ add a comment text to the archive
 manually specify the archive creation date/time (yyyy\-mm\-ddThh:mm:ss[(+|\-)HH:MM] format, (+|\-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory.
 .TP
 .BI \-\-chunker\-params \ PARAMS
-specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: buzhash,19,23,21,4095
+specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, NC_LEVEL). default: fastcdc,19,23,21,2
 .TP
 .BI \-C \ COMPRESSION\fR,\fB \ \-\-compression \ COMPRESSION
 select compression algorithm, see the output of the \(dqborg help compression\(dq command for details.
@@ -358,7 +361,7 @@ $ fusermount \-u sshfs\-mount
 
 # Make a big effort in fine\-grained deduplication (big chunk management
 # overhead, needs a lot of RAM and disk space; see the formula in the internals docs):
-$ borg create \-\-chunker\-params buzhash,10,23,16,4095 small /smallstuff
+$ borg create \-\-chunker\-params fastcdc,10,23,16,2 small /smallstuff
 
 # Backup a raw device (must not be active/in use/mounted at that time)
 $ borg create \-\-read\-special \-\-chunker\-params fixed,4194304 my\-sdx /dev/sdX
diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc
index d53f695388..b914e9f204 100644
--- a/docs/usage/create.rst.inc
+++ b/docs/usage/create.rst.inc
@@ -12,103 +12,105 @@ borg create
 
     .. class:: borg-options-table
 
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    | **positional arguments**                                                                                                                                                                                                                                                                                      |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``NAME``                                          | specify the archive name                                                                                                                                                                          |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``PATH``                                          | paths to archive                                                                                                                                                                                  |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    | **options**                                                                                                                                                                                                                                                                                                   |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``-n``, ``--dry-run``                             | do not create a backup archive                                                                                                                                                                    |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``-s``, ``--stats``                               | print statistics for the created archive                                                                                                                                                          |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--list``                                        | output a verbose list of items (files, dirs, ...)                                                                                                                                                 |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--filter STATUSCHARS``                          | only display items with the given status characters (see description)                                                                                                                             |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--json``                                        | output stats as JSON. Implies ``--stats``.                                                                                                                                                        |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--stdin-name NAME``                             | use NAME in archive for stdin data (default: 'stdin')                                                                                                                                             |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--stdin-user USER``                             | set user USER in archive for stdin data (default: do not store user/uid)                                                                                                                          |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--stdin-group GROUP``                           | set group GROUP in archive for stdin data (default: do not store group/gid)                                                                                                                       |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--stdin-mode M``                                | set mode to M in archive for stdin data (default: 0660)                                                                                                                                           |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--content-from-command``                        | interpret PATH as a command and store its stdout. See also the section 'Reading from stdin' below.                                                                                                |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--paths-from-stdin``                            | read DELIM-separated list of paths to back up from stdin. All control is external: it will back up all files given - no more, no less.                                                            |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--paths-from-command``                          | interpret PATH as command and treat its output as ``--paths-from-stdin``                                                                                                                          |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--paths-from-shell-command``                    | interpret PATH as shell command and treat its output as ``--paths-from-stdin``                                                                                                                    |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--paths-delimiter DELIM``                       | set path delimiter for ``--paths-from-stdin`` and ``--paths-from-command`` (default: ``\n``)                                                                                                      |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    | .. class:: borg-common-opt-ref                                                                                                                                                                                                                                                                                |
-    |                                                                                                                                                                                                                                                                                                               |
-    | :ref:`common_options`                                                                                                                                                                                                                                                                                         |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    | **Include/Exclude options**                                                                                                                                                                                                                                                                                   |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``-e PATTERN``, ``--exclude PATTERN``             | exclude paths matching PATTERN                                                                                                                                                                    |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--exclude-from EXCLUDEFILE``                    | read exclude patterns from EXCLUDEFILE, one per line                                                                                                                                              |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--pattern PATTERN``                             | include/exclude paths matching PATTERN                                                                                                                                                            |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--patterns-from PATTERNFILE``                   | read include/exclude patterns from PATTERNFILE, one per line                                                                                                                                      |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--exclude-caches``                              | exclude directories that contain a CACHEDIR.TAG file (https://www.bford.info/cachedir/spec.html)                                                                                                  |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--exclude-if-present NAME``                     | exclude directories that are tagged by containing a filesystem object with the given NAME                                                                                                         |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--keep-exclude-tags``                           | if tag objects are specified with ``--exclude-if-present``, do not omit the tag objects themselves from the backup archive                                                                        |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--exclude-dataless``                            | exclude files flagged DATALESS (macOS: placeholder files whose content is not materialized locally, e.g. not-downloaded cloud storage files)                                                      |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    | **Filesystem options**                                                                                                                                                                                                                                                                                        |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``-x``, ``--one-file-system``                     | stay in the same file system and do not store mount points of other file systems - this might behave different from your expectations, see the description below.                                 |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--numeric-ids``                                 | only store numeric user and group identifiers                                                                                                                                                     |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--atime``                                       | do store atime into archive                                                                                                                                                                       |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--noctime``                                     | do not store ctime into archive                                                                                                                                                                   |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--nobirthtime``                                 | do not store birthtime (creation date) into archive                                                                                                                                               |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--noflags``                                     | do not read and store flags (e.g. NODUMP, IMMUTABLE) into archive                                                                                                                                 |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--noacls``                                      | do not read and store ACLs into archive                                                                                                                                                           |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--noxattrs``                                    | do not read and store xattrs into archive                                                                                                                                                         |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--sparse``                                      | detect sparse holes in input (supported only by fixed chunker)                                                                                                                                    |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--files-cache MODE``                            | operate files cache in MODE. default: ctime,size,inode                                                                                                                                            |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--files-changed MODE``                          | specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).                                |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--read-special``                                | open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.                                                 |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    | **Archive options**                                                                                                                                                                                                                                                                                           |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--comment COMMENT``                             | add a comment text to the archive                                                                                                                                                                 |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--timestamp TIMESTAMP``                         | manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, (+|-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory. |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--chunker-params PARAMS``                       | specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, NC_LEVEL). default: fastcdc,19,23,21,2                                                                        |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``-C COMPRESSION``, ``--compression COMPRESSION`` | select compression algorithm, see the output of the "borg help compression" command for details.                                                                                                  |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--tags TAG``                                    | add tags to archive (comma-separated or multiple arguments)                                                                                                                                       |
-    +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **positional arguments**                                                                                                                                                                                                                                                                                                                                   |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``NAME``                                          | specify the archive name                                                                                                                                                                                                                       |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``PATH``                                          | paths to archive                                                                                                                                                                                                                               |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **options**                                                                                                                                                                                                                                                                                                                                                |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``-n``, ``--dry-run``                             | do not create a backup archive                                                                                                                                                                                                                 |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``-s``, ``--stats``                               | print statistics for the created archive                                                                                                                                                                                                       |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--list``                                        | output a verbose list of items (files, dirs, ...)                                                                                                                                                                                              |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--filter STATUSCHARS``                          | only display items with the given status characters (see description)                                                                                                                                                                          |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--json``                                        | output stats as JSON. Implies ``--stats``.                                                                                                                                                                                                     |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--stdin-name NAME``                             | use NAME in archive for stdin data (default: 'stdin')                                                                                                                                                                                          |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--stdin-user USER``                             | set user USER in archive for stdin data (default: do not store user/uid)                                                                                                                                                                       |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--stdin-group GROUP``                           | set group GROUP in archive for stdin data (default: do not store group/gid)                                                                                                                                                                    |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--stdin-mode M``                                | set mode to M in archive for stdin data (default: 0660)                                                                                                                                                                                        |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--content-from-command``                        | interpret PATH as a command and store its stdout. See also the section 'Reading from stdin' below.                                                                                                                                             |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--paths-from-stdin``                            | read DELIM-separated list of paths to back up from stdin. All control is external: it will back up all files given - no more, no less.                                                                                                         |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--paths-from-command``                          | interpret PATH as command and treat its output as ``--paths-from-stdin``                                                                                                                                                                       |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--paths-from-shell-command``                    | interpret PATH as shell command and treat its output as ``--paths-from-stdin``                                                                                                                                                                 |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--paths-delimiter DELIM``                       | set path delimiter for ``--paths-from-stdin`` and ``--paths-from-command`` (default: ``\n``)                                                                                                                                                   |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | .. class:: borg-common-opt-ref                                                                                                                                                                                                                                                                                                                             |
+    |                                                                                                                                                                                                                                                                                                                                                            |
+    | :ref:`common_options`                                                                                                                                                                                                                                                                                                                                      |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **Include/Exclude options**                                                                                                                                                                                                                                                                                                                                |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``-e PATTERN``, ``--exclude PATTERN``             | exclude paths matching PATTERN                                                                                                                                                                                                                 |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--exclude-from EXCLUDEFILE``                    | read exclude patterns from EXCLUDEFILE, one per line                                                                                                                                                                                           |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--pattern PATTERN``                             | include/exclude paths matching PATTERN                                                                                                                                                                                                         |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--patterns-from PATTERNFILE``                   | read include/exclude patterns from PATTERNFILE, one per line                                                                                                                                                                                   |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--exclude-caches``                              | exclude directories that contain a CACHEDIR.TAG file (https://www.bford.info/cachedir/spec.html)                                                                                                                                               |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--exclude-if-present NAME``                     | exclude directories that are tagged by containing a filesystem object with the given NAME                                                                                                                                                      |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--keep-exclude-tags``                           | if tag objects are specified with ``--exclude-if-present``, do not omit the tag objects themselves from the backup archive                                                                                                                     |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--exclude-dataless``                            | exclude files flagged DATALESS (macOS: placeholder files whose content is not materialized locally, e.g. not-downloaded cloud storage files)                                                                                                   |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **Filesystem options**                                                                                                                                                                                                                                                                                                                                     |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``-x``, ``--one-file-system``                     | stay in the same file system and do not store mount points of other file systems - this might behave different from your expectations, see the description below.                                                                              |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--numeric-ids``                                 | only store numeric user and group identifiers                                                                                                                                                                                                  |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--atime``                                       | do store atime into archive                                                                                                                                                                                                                    |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--noctime``                                     | do not store ctime into archive                                                                                                                                                                                                                |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--nobirthtime``                                 | do not store birthtime (creation date) into archive                                                                                                                                                                                            |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--noflags``                                     | do not read and store flags (e.g. NODUMP, IMMUTABLE) into archive                                                                                                                                                                              |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--noacls``                                      | do not read and store ACLs into archive                                                                                                                                                                                                        |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--noxattrs``                                    | do not read and store xattrs into archive                                                                                                                                                                                                      |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--sparse``                                      | detect sparse holes in input (supported only by fixed chunker)                                                                                                                                                                                 |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--files-cache MODE``                            | operate files cache in MODE. default: ctime,size,inode                                                                                                                                                                                         |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--files-changed MODE``                          | specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).                                                                             |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--read-special``                                | open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.                                                                                              |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--read-special-timeout SECONDS``                | when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **Archive options**                                                                                                                                                                                                                                                                                                                                        |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--comment COMMENT``                             | add a comment text to the archive                                                                                                                                                                                                              |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--timestamp TIMESTAMP``                         | manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, (+|-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory.                                              |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--chunker-params PARAMS``                       | specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, NC_LEVEL). default: fastcdc,19,23,21,2                                                                                                                     |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``-C COMPRESSION``, ``--compression COMPRESSION`` | select compression algorithm, see the output of the "borg help compression" command for details.                                                                                                                                               |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--tags TAG``                                    | add tags to archive (comma-separated or multiple arguments)                                                                                                                                                                                    |
+    +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 
     .. raw:: html
 
@@ -170,6 +172,7 @@ borg create
         --files-cache MODE        operate files cache in MODE. default: ctime,size,inode
         --files-changed MODE      specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).
         --read-special            open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.
+        --read-special-timeout SECONDS    when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds.
 
 
     Archive options
diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst
index b79410ecd1..d6f846ff3c 100644
--- a/docs/usage/notes.rst
+++ b/docs/usage/notes.rst
@@ -119,6 +119,18 @@ block devices, like e.g. LVM snapshots or inactive LVs or disk partitions.
 You need to be careful about what you include when using ``--read-special``,
 e.g. if you include ``/dev/zero``, your backup will never terminate.
 
+Reading from a FIFO blocks until a writer connects and sends data - if that
+never happens (e.g. due to a broken producer script), ``borg create`` would
+appear to hang. Thus, when no data arrives from a FIFO or character device
+for ``--read-special-timeout SECONDS`` (default: 1800, i.e. 30 minutes; this
+includes waiting for a FIFO's writer to connect), borg gives up on that file:
+it is skipped with an error and the backup continues with the remaining
+files. Choose a timeout larger than the pauses a legitimate (slow) producer
+makes while sending data - or give 0 to disable the timeout and wait forever.
+Note: with the timeout active, a writer that connects, but closes without
+sending anything, is reported as a timeout, too (instead of storing a file
+with empty content).
+
 Restoring such files' content is currently only supported one at a time via
 ``--stdout`` option (and you have to redirect stdout to where ever it shall go,
 maybe directly into an existing device file of your choice or indirectly via
diff --git a/src/borg/archive.py b/src/borg/archive.py
index 196d20f2e5..db7de5cd03 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -29,7 +29,7 @@
 from .crypto.low_level import IntegrityError as IntegrityErrorBase
 from .helpers import BackupError, BackupRaceConditionError, BackupItemExcluded
 from .helpers import BackupSymlinkParentError, BackupPathTraversalError
-from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError
+from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError, BackupTimeoutError
 from .helpers import HardLinkManager
 from .helpers import ChunkIteratorFileWrapper, open_item
 from .helpers import Error, IntegrityError, set_ec, sig_int
@@ -42,7 +42,7 @@
 from .helpers import bin_to_hex
 from .helpers import safe_ns
 from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi, get_progress_dt
-from .helpers import os_open, flags_normal, flags_dir
+from .helpers import os_open, flags_normal, flags_dir, O_, SpecialFileReader
 from .helpers import os_stat
 from .helpers import msgpack
 from .helpers.lrucache import LRUCache
@@ -267,6 +267,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
                 errno.EBUSY: BackupPermissionError,
                 errno.ENOENT: BackupFileNotFoundError,
                 errno.EIO: BackupIOError,
+                errno.ETIMEDOUT: BackupTimeoutError,
             }
             e_cls = E_MAP.get(exc_val.errno, BackupOSError)
             raise e_cls(self.op, exc_val) from exc_val
@@ -1370,6 +1371,7 @@ def __init__(
         log_json,
         file_status_printer=None,
         files_changed="mtime" if is_win32 else "ctime",
+        read_special_timeout=None,
     ):
         self.metadata_collector = metadata_collector
         self.cache = cache
@@ -1379,6 +1381,7 @@ def __init__(
         self.show_progress = show_progress
         self.print_file_status = file_status_printer or (lambda *args: None)
         self.files_changed = files_changed
+        self.read_special_timeout = read_special_timeout
 
         self.hlm = HardLinkManager(id_type=tuple, info_type=(list, type(None)))  # (dev, ino) -> chunks or None
         self.stats = Statistics(output_json=log_json)  # threading: done by cache (including progress)
@@ -1531,6 +1534,15 @@ def process_pipe(self, *, path, cache, fd, mode, user=None, group=None):
         return status
 
     def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, last_try=False, strip_prefix):
+        if self.read_special_timeout is not None and (stat.S_ISFIFO(st.st_mode) or stat.S_ISCHR(st.st_mode)):
+            # timeout-limited reading of fifos / char devices (--read-special-timeout).
+            # O_NONBLOCK makes opening a fifo succeed immediately even without a connected
+            # writer - waiting for a writer / for data is then subject to the timeout,
+            # see SpecialFileReader.
+            read_special_timeout = self.read_special_timeout
+            flags |= O_("NONBLOCK")
+        else:
+            read_special_timeout = None
         with self.create_helper(path, st, None, strip_prefix=strip_prefix) as (
             item,
             status,
@@ -1600,14 +1612,15 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal,
                         # and still commit the archive -- referencing chunks that were never durably
                         # stored. An unwrapped repository OSError is critical and aborts create before
                         # archive.save() runs (see the BackupOSError docstring).
-                        self.process_file_chunks(
-                            item,
-                            cache,
-                            self.stats,
-                            self.show_progress,
+                        if read_special_timeout is not None:
+                            # all reads go through the timeout-enforcing wrapper (fh stays unused).
+                            chunk_iter = self.chunker.chunkify(SpecialFileReader(fd, read_special_timeout), st=st)
+                        else:
                             # passing st saves FileReader a stat call; regular files take the
                             # direct read path, special files (--read-special) the buffered one.
-                            backup_io_iter(self.chunker.chunkify(None, fd, st=st)),
+                            chunk_iter = self.chunker.chunkify(None, fd, st=st)
+                        self.process_file_chunks(
+                            item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter)
                         )
                         self.stats.chunking_time = self.chunker.chunking_time
                         end_reading = time.time_ns()
diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py
index 87f183880a..1727ca14d2 100644
--- a/src/borg/archiver/create_cmd.py
+++ b/src/borg/archiver/create_cmd.py
@@ -16,7 +16,7 @@
 from ..cache import Cache
 from ..constants import *  # NOQA
 from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec
-from ..helpers import archivename_validator, FilesCacheMode, octal_int
+from ..helpers import archivename_validator, FilesCacheMode, octal_int, nonnegative_seconds
 from ..helpers import eval_escapes
 from ..helpers import timestamp, archive_ts_now
 from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify
@@ -43,6 +43,13 @@ class CreateMixIn:
     @with_repository(compatibility=(Manifest.Operation.WRITE,))
     def do_create(self, args, repository, manifest):
         """Creates a new archive."""
+        if args.read_special_timeout is not None and not args.read_special:
+            raise CommandError("--read-special-timeout requires --read-special.")
+        read_special_timeout = args.read_special_timeout
+        if read_special_timeout is None:
+            read_special_timeout = READ_SPECIAL_TIMEOUT_DEFAULT
+        if read_special_timeout == 0:
+            read_special_timeout = None  # wait forever
         key = manifest.key
         matcher = PatternMatcher(fallback=True)
         matcher.add_inclexcl(args.patterns)
@@ -268,6 +275,7 @@ def create_inner(archive, cache, fso):
                     log_json=args.log_json,
                     file_status_printer=self.print_file_status,
                     files_changed=args.files_changed,
+                    read_special_timeout=read_special_timeout,
                 )
                 create_inner(archive, cache, fso)
             args.stats |= args.json
@@ -395,8 +403,9 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                 return "-"
             except BackupError as err:
                 if isinstance(err, BackupOSError):
-                    if err.errno in (errno.EPERM, errno.EACCES):
-                        # Do not try again, such errors can not be fixed by retrying.
+                    if err.errno in (errno.EPERM, errno.EACCES, errno.ETIMEDOUT):
+                        # Do not try again: permission errors can not be fixed by retrying
+                        # and a --read-special-timeout would just expire again (10 more times).
                         raise
                 if last_try:
                     # giving up with retries, error will be dealt with (logged) by upper error handler
@@ -982,6 +991,18 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
             help="open and read block and char device files as well as FIFOs as if they were "
             "regular files. Also follows symlinks pointing to these kinds of files.",
         )
+        fs_group.add_argument(
+            "--read-special-timeout",
+            metavar="SECONDS",
+            dest="read_special_timeout",
+            type=nonnegative_seconds,
+            default=None,
+            action=Highlander,
+            help="when reading from FIFOs or character devices (see --read-special): skip the "
+            "file with an error if no data arrives for more than SECONDS (this includes waiting "
+            "for a FIFO's writer to connect). Give 0 to wait forever. default: %d seconds."
+            % READ_SPECIAL_TIMEOUT_DEFAULT,
+        )
 
         archive_group = subparser.add_argument_group("Archive options")
         archive_group.add_argument(
diff --git a/src/borg/constants.py b/src/borg/constants.py
index 44a9e18734..5f6ff8effa 100644
--- a/src/borg/constants.py
+++ b/src/borg/constants.py
@@ -38,6 +38,11 @@
 # forcing to 0o100XXX later
 STDIN_MODE_DEFAULT = 0o660
 
+# default for --read-special-timeout [s]: how long reading a fifo / char device may wait
+# for data to arrive (incl. waiting for a fifo's writer to connect) before the file is
+# skipped with an error. 0 means "wait forever".
+READ_SPECIAL_TIMEOUT_DEFAULT = 1800.0
+
 # RepoObj types
 ROBJ_MANIFEST = "M"  # Manifest (directory of archives, other metadata) object
 ROBJ_ARCHIVE_META = "A"  # main archive metadata object
diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py
index 21d2e764c2..7a0fdbccf5 100644
--- a/src/borg/helpers/__init__.py
+++ b/src/borg/helpers/__init__.py
@@ -16,19 +16,20 @@
 from .errors import RTError, PathNotAllowed, modern_ec
 from .errors import BorgWarning, FileChangedWarning, BackupWarning, IncludePatternNeverMatchedWarning
 from .errors import BackupError, BackupOSError, BackupRaceConditionError, BackupItemExcluded
-from .errors import BackupPermissionError, BackupIOError, BackupFileNotFoundError
+from .errors import BackupPermissionError, BackupIOError, BackupFileNotFoundError, BackupTimeoutError
 from .errors import BackupSymlinkParentError, BackupPathTraversalError, BackupHardlinkSourceError
 from .fs import ensure_dir, join_base_dir
 from .fs import get_security_dir, get_keys_dir, get_base_dir, get_cache_dir, get_config_dir, get_runtime_dir
 from .fs import dir_is_tagged, dir_is_cachedir, remove_dotdot_prefixes, make_path_safe, scandir_inorder
 from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount, slashify
+from .fs import SpecialFileReader
 from .fs import O_, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime
 from .fs import HardLinkManager
 from .misc import sysinfo, log_multi, consume
 from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
 from .parseformat import octal_int, bin_to_hex, hex_to_bin, safe_encode, safe_decode
 from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd
-from .parseformat import eval_escapes, decode_dict, interval, int_or_interval
+from .parseformat import eval_escapes, decode_dict, interval, int_or_interval, nonnegative_seconds
 from .parseformat import (
     PathSpec,
     FilesystemPathSpec,
diff --git a/src/borg/helpers/errors.py b/src/borg/helpers/errors.py
index 5ed00432ef..72993efb7b 100644
--- a/src/borg/helpers/errors.py
+++ b/src/borg/helpers/errors.py
@@ -217,5 +217,13 @@ class BackupHardlinkSourceError(BackupError):
     exit_mcode = 110
 
 
+class BackupTimeoutError(BackupOSError):
+    """{}: {}"""
+
+    # Timed out waiting for data while reading from a special file (fifo / character device),
+    # see the --read-special-timeout option of borg create.
+    exit_mcode = 111
+
+
 class BackupItemExcluded(Exception):
     """Used internally to skip an item from processing when it is excluded."""
diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py
index c32418d722..1412416450 100644
--- a/src/borg/helpers/fs.py
+++ b/src/borg/helpers/fs.py
@@ -3,10 +3,12 @@
 import os
 import posixpath
 import re
+import select
 import stat
 import subprocess
 import sys
 import textwrap
+import time
 from pathlib import Path
 
 import platformdirs
@@ -574,6 +576,68 @@ def os_stat(*, path=None, parent_fd=None, name=None, follow_symlinks=False):
     return os.stat(fname, dir_fd=parent_fd, follow_symlinks=follow_symlinks)
 
 
+class SpecialFileReader:
+    """
+    File-like reader for special files (fifos / character devices) opened with O_NONBLOCK,
+    enforcing a timeout while waiting for data.
+
+    The timeout limits how long a read waits for data to *arrive*: as long as data keeps
+    flowing with gaps shorter than the timeout, reading never times out. When the timeout
+    is exceeded, OSError(ETIMEDOUT) is raised.
+
+    This also covers waiting for a fifo's writer to connect: opening a fifo read-only with
+    O_NONBLOCK succeeds immediately even when there is no writer (yet) and until one connects,
+    os.read() returns b"" (which otherwise means EOF). Thus, b"" only counts as EOF after some
+    data was received before - until then, it means "no writer connected (yet)" and we keep
+    waiting. Consequence: a writer connecting, but closing without writing anything, results
+    in a timeout, not in empty file content.
+    """
+
+    NO_WRITER_POLL_INTERVAL = 0.1  # [s] how often to re-check a fifo nobody has opened for writing
+
+    def __init__(self, fd, timeout):
+        self.fd = fd  # OS-level file descriptor, opened with O_RDONLY | O_NONBLOCK
+        self.timeout = timeout  # [s] maximum time to wait for data to arrive
+        self.got_data = False  # whether any data was received yet
+
+    def read(self, size):
+        # Loop until we have <size> bytes, EOF or a timeout: the chunker's FileReader expects
+        # short reads only at EOF, and a single os.read from a fifo returns at most the pipe
+        # buffer's current content.
+        parts = []
+        remaining = size
+        deadline = time.monotonic() + self.timeout
+        while remaining:
+            try:
+                data = os.read(self.fd, remaining)
+            except BlockingIOError:
+                data = None  # a writer is connected, but no data is available right now
+            if data:
+                parts.append(data)
+                remaining -= len(data)
+                self.got_data = True
+                deadline = time.monotonic() + self.timeout
+                continue
+            if data == b"" and self.got_data:
+                break  # EOF - all writers closed after having sent data.
+            wait = deadline - time.monotonic()
+            if wait <= 0.0:
+                raise OSError(errno.ETIMEDOUT, f"no data received for more than {self.timeout} seconds")
+            if data is None:
+                # wait for data to arrive (select returns early when it does or when writers close)
+                select.select([self.fd], [], [], wait)
+            else:
+                # fifo without a connected writer: there is nothing to select() on that would
+                # signal a writer connecting, so re-check periodically.
+                time.sleep(min(self.NO_WRITER_POLL_INTERVAL, wait))
+        return parts[0] if len(parts) == 1 else b"".join(parts)
+
+    def seek(self, pos, whence=os.SEEK_SET):
+        # Special files are not seekable - raise what lseek on a fifo fd would raise
+        # (the sparse file support probes seekability and handles OSError).
+        raise OSError(errno.ESPIPE, os.strerror(errno.ESPIPE))
+
+
 def umount(mountpoint):
     from . import set_ec
 
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index 7c6e920d86..fe91fc7cd8 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -129,6 +129,17 @@ def decode_dict(d, keys, encoding="utf-8", errors="surrogateescape"):
     return d
 
 
+def nonnegative_seconds(s):
+    """Parse a nonnegative number of seconds (int or float, e.g. ``0``, ``30`` or ``2.5``)."""
+    try:
+        seconds = float(s)
+    except ValueError:
+        raise ArgumentTypeError(f"invalid seconds value: {s!r}") from None
+    if not seconds >= 0.0:
+        raise ArgumentTypeError("seconds value must be >= 0")
+    return seconds
+
+
 def interval(s):
     """Parse an interval string (e.g. ``7d``, ``2w``, ``30M``) into a timedelta.
 
diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py
index d0d4df7ecc..69995509d0 100644
--- a/src/borg/testsuite/archiver/create_cmd_test.py
+++ b/src/borg/testsuite/archiver/create_cmd_test.py
@@ -16,7 +16,7 @@
 from ...platform import is_win32, is_cygwin
 from ...platformflags import is_msystem
 from ...repository import Repository
-from ...helpers import CommandError, BackupPermissionError
+from ...helpers import CommandError, BackupPermissionError, BackupTimeoutError
 from .. import has_lchflags, has_mknod
 from .. import changedir
 from .. import (
@@ -954,6 +954,150 @@ def test_create_topical(archivers, request):
     assert "file1" in output
 
 
+def _drain_fifo_and_join(fifo_fn, thread, size=65536):
+    """In case `borg create` failed to open/read the FIFO, unblock the feeder thread."""
+    fd = os.open(fifo_fn, os.O_RDONLY | os.O_NONBLOCK)
+    try:
+        while os.read(fd, size):
+            pass
+    except OSError:
+        pass
+    finally:
+        os.close(fd)
+    thread.join()
+
+
+@pytest.mark.skipif(not are_fifos_supported() or is_cygwin, reason="FIFOs not supported, hangs on cygwin")
+@pytest.mark.parametrize("timeout_args", ([], ["--read-special-timeout=0"]), ids=["default-timeout", "no-timeout"])
+def test_create_read_special_big_fifo(archivers, request, timeout_args):
+    # os.read() on a fifo returns at most the pipe buffer's content (often just 64 KiB),
+    # usually much less than the chunker requests per read - especially with a writer
+    # that pauses between bursts. borg must keep reading and not treat a short read as
+    # EOF, else the archived fifo content gets truncated.
+    # Parametrization covers both read paths: the SpecialFileReader wrapper (default
+    # timeout) and the plain blocking reads (--read-special-timeout=0 == wait forever).
+    archiver = request.getfixturevalue(archivers)
+    import time
+    from threading import Thread
+
+    data = b"".join(length.to_bytes(4, "big") * 1024 for length in range(512))  # 2 MiB, way > pipe buffer
+
+    def fifo_feeder(fifo_fn, data):
+        fd = os.open(fifo_fn, os.O_WRONLY)
+        try:
+            burst_size = len(data) // 4
+            for start in range(0, len(data), burst_size):
+                burst = data[start : start + burst_size]
+                pos = 0
+                while pos < len(burst):
+                    pos += os.write(fd, burst[pos:])
+                time.sleep(0.1)  # writer pause: reads on the other side come up short
+        except BrokenPipeError:
+            pass  # borg closed the fifo prematurely (that is the bug this test is about)
+        finally:
+            os.close(fd)
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    fifo_fn = os.path.join(archiver.input_path, "fifo")
+    os.mkfifo(fifo_fn)
+    t = Thread(target=fifo_feeder, args=(fifo_fn, data))
+    t.start()
+    try:
+        cmd(archiver, "create", "--read-special", *timeout_args, "test", "input/fifo")
+    finally:
+        _drain_fifo_and_join(fifo_fn, t)
+    with changedir("output"):
+        cmd(archiver, "extract", "test")
+        with open("input/fifo", "rb") as f:
+            extracted_data = f.read()
+    assert len(extracted_data) == len(data)
+    assert extracted_data == data
+
+
+@pytest.mark.skipif(not are_fifos_supported() or is_cygwin, reason="FIFOs not supported, hangs on cygwin")
+def test_create_read_special_timeout_expired(archivers, request):
+    # a fifo nobody ever opens for writing: borg must give up after --read-special-timeout,
+    # skip the fifo with an error and still back up the other files.
+    archiver = request.getfixturevalue(archivers)
+    import time
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file1", size=1024)
+    os.mkfifo(os.path.join(archiver.input_path, "fifo"))
+    expected_ec = BackupTimeoutError("read", OSError(errno.ETIMEDOUT, "timeout")).exit_code
+    if expected_ec == EXIT_ERROR:  # workaround, TODO: fix it
+        expected_ec = EXIT_WARNING
+    started = time.monotonic()
+    out = cmd(
+        archiver,
+        "create",
+        "--read-special",
+        "--read-special-timeout=1",
+        "test",
+        "input",
+        exit_code=expected_ec,  # WARNING status: could not back up the fifo.
+    )
+    assert time.monotonic() - started < 30  # timed out, no eternal hang
+    assert "retry: 1 of " not in out  # timeouts are NOT retried
+    listing = cmd(archiver, "list", "test", "--format={path}{NL}")
+    assert "input/file1" in listing
+    assert "fifo" not in listing
+
+
+def test_create_read_special_timeout_requires_read_special(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    if archiver.FORK_DEFAULT:
+        expected_ec = CommandError().exit_code
+        output = cmd(archiver, "create", "--read-special-timeout=5", "test", "input", exit_code=expected_ec)
+        assert "--read-special-timeout requires --read-special." in output
+    else:
+        with pytest.raises(CommandError):
+            cmd(archiver, "create", "--read-special-timeout=5", "test", "input")
+
+
+@pytest.mark.skipif(not are_fifos_supported() or is_cygwin, reason="FIFOs not supported, hangs on cygwin")
+def test_create_read_special_timeout_slow_writer(archivers, request):
+    # a writer that connects late and pauses between bursts must not trigger the timeout,
+    # as long as the pauses stay below --read-special-timeout.
+    archiver = request.getfixturevalue(archivers)
+    import time
+    from threading import Thread
+
+    data = b"".join(length.to_bytes(4, "big") * 256 for length in range(64))  # 64 KiB
+    burst_size = len(data) // 4
+
+    def fifo_feeder(fifo_fn, data):
+        time.sleep(0.3)  # writer connects late
+        fd = os.open(fifo_fn, os.O_WRONLY)
+        try:
+            for start in range(0, len(data), burst_size):
+                burst = data[start : start + burst_size]
+                pos = 0
+                while pos < len(burst):
+                    pos += os.write(fd, burst[pos:])
+                time.sleep(0.2)  # data flows with gaps < timeout
+        except BrokenPipeError:
+            pass  # borg gave up early - the asserts below will complain
+        finally:
+            os.close(fd)
+
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    fifo_fn = os.path.join(archiver.input_path, "fifo")
+    os.mkfifo(fifo_fn)
+    t = Thread(target=fifo_feeder, args=(fifo_fn, data))
+    t.start()
+    try:
+        cmd(archiver, "create", "--read-special", "--read-special-timeout=30", "test", "input/fifo")
+    finally:
+        _drain_fifo_and_join(fifo_fn, t)
+    with changedir("output"):
+        cmd(archiver, "extract", "test")
+        with open("input/fifo", "rb") as f:
+            extracted_data = f.read()
+    assert extracted_data == data
+
+
 @pytest.mark.skipif(not are_fifos_supported() or is_cygwin, reason="FIFOs not supported, hangs on cygwin")
 def test_create_read_special_symlink(archivers, request):
     archiver = request.getfixturevalue(archivers)
diff --git a/src/borg/testsuite/helpers/fs_test.py b/src/borg/testsuite/helpers/fs_test.py
index 2eec3de238..e67a230013 100644
--- a/src/borg/testsuite/helpers/fs_test.py
+++ b/src/borg/testsuite/helpers/fs_test.py
@@ -23,9 +23,10 @@
     remove_dotdot_prefixes,
     make_path_safe,
     map_chars,
+    SpecialFileReader,
 )
 from ...platform import is_win32, is_darwin
-from .. import are_hardlinks_supported
+from .. import are_fifos_supported, are_hardlinks_supported
 from .. import rejected_dotdot_paths
 
 
@@ -468,3 +469,140 @@ def test_map_chars(monkeypatch):
     assert map_chars("foo|bar") == "foo\uf07cbar"
     assert map_chars("foo?bar") == "foo\uf03fbar"
     assert map_chars("foo*bar") == "foo\uf02abar"
+
+
+posix_only = pytest.mark.skipif(is_win32, reason="POSIX-only (non-blocking pipe reads, select)")
+
+
+@contextmanager
+def nonblocking_pipe():
+    r, w = os.pipe()
+    os.set_blocking(r, False)  # SpecialFileReader expects an O_NONBLOCK fd
+    try:
+        yield r, w
+    finally:
+        os.close(r)
+        try:
+            os.close(w)
+        except OSError:
+            pass  # feeder already closed it
+
+
+@posix_only
+def test_special_file_reader_loops_until_filled():
+    # a single os.read from a pipe returns at most the pipe buffer's content -
+    # read() must loop until it has the requested amount (or EOF).
+    import time
+    from threading import Thread
+
+    data = bytes(range(256)) * 1024  # 256 kiB, > default pipe buffer
+
+    with nonblocking_pipe() as (r, w):
+
+        def feeder():
+            burst_size = len(data) // 4
+            for start in range(0, len(data), burst_size):
+                burst = data[start : start + burst_size]
+                pos = 0
+                while pos < len(burst):
+                    pos += os.write(w, burst[pos:])
+                time.sleep(0.05)  # make sure the reader sees short reads
+            os.close(w)
+
+        t = Thread(target=feeder)
+        t.start()
+        try:
+            reader = SpecialFileReader(r, timeout=5.0)
+            assert reader.read(len(data)) == data
+            assert reader.read(1024) == b""  # EOF
+        finally:
+            t.join()
+
+
+@posix_only
+def test_special_file_reader_eof():
+    with nonblocking_pipe() as (r, w):
+        os.write(w, b"data")
+        os.close(w)
+        reader = SpecialFileReader(r, timeout=5.0)
+        assert reader.read(1024) == b"data"  # short read at EOF
+        assert reader.read(1024) == b""
+
+
+@posix_only
+def test_special_file_reader_timeout_silent_writer():
+    # a connected writer that never sends anything must trigger the timeout.
+    import time
+
+    with nonblocking_pipe() as (r, w):
+        reader = SpecialFileReader(r, timeout=0.2)
+        started = time.monotonic()
+        with pytest.raises(OSError) as exc_info:
+            reader.read(1024)
+        assert exc_info.value.errno == errno.ETIMEDOUT
+        assert 0.1 < time.monotonic() - started < 5.0
+
+
+@posix_only
+def test_special_file_reader_timeout_stalled_writer():
+    # a writer stalling mid-stream (after data was already received) must trigger the timeout, too.
+    with nonblocking_pipe() as (r, w):
+        os.write(w, b"data")
+        reader = SpecialFileReader(r, timeout=0.2)
+        with pytest.raises(OSError) as exc_info:
+            reader.read(1024)  # got 4 bytes, then nothing more
+        assert exc_info.value.errno == errno.ETIMEDOUT
+
+
+@pytest.mark.skipif(not are_fifos_supported(), reason="FIFOs not supported")
+def test_special_file_reader_fifo_no_writer(tmp_path):
+    # opening a fifo with O_NONBLOCK succeeds without a writer and os.read returns b"" -
+    # that must count as "waiting for a writer" (-> timeout), not as EOF (-> empty file).
+    fifo_fn = str(tmp_path / "fifo")
+    os.mkfifo(fifo_fn)
+    fd = os.open(fifo_fn, os.O_RDONLY | os.O_NONBLOCK)
+    try:
+        reader = SpecialFileReader(fd, timeout=0.3)
+        with pytest.raises(OSError) as exc_info:
+            reader.read(1024)
+        assert exc_info.value.errno == errno.ETIMEDOUT
+    finally:
+        os.close(fd)
+
+
+@pytest.mark.skipif(not are_fifos_supported(), reason="FIFOs not supported")
+def test_special_file_reader_fifo_late_writer(tmp_path):
+    # a writer connecting late (but within the timeout) must not trigger the timeout.
+    import time
+    from threading import Thread
+
+    fifo_fn = str(tmp_path / "fifo")
+    os.mkfifo(fifo_fn)
+
+    def feeder():
+        time.sleep(0.2)
+        wfd = os.open(fifo_fn, os.O_WRONLY)
+        try:
+            os.write(wfd, b"late data")
+        finally:
+            os.close(wfd)
+
+    fd = os.open(fifo_fn, os.O_RDONLY | os.O_NONBLOCK)
+    t = Thread(target=feeder)
+    t.start()
+    try:
+        reader = SpecialFileReader(fd, timeout=5.0)
+        assert reader.read(1024) == b"late data"
+        assert reader.read(1024) == b""  # EOF
+    finally:
+        t.join()
+        os.close(fd)
+
+
+@posix_only
+def test_special_file_reader_not_seekable():
+    with nonblocking_pipe() as (r, w):
+        reader = SpecialFileReader(r, timeout=1.0)
+        with pytest.raises(OSError) as exc_info:
+            reader.seek(0)
+        assert exc_info.value.errno == errno.ESPIPE
diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py
index 9c47d8153e..fa95f8e8d4 100644
--- a/src/borg/testsuite/helpers/parseformat_test.py
+++ b/src/borg/testsuite/helpers/parseformat_test.py
@@ -21,6 +21,7 @@
     parse_file_size,
     interval,
     int_or_interval,
+    nonnegative_seconds,
     partial_format,
     clean_lines,
     format_line,
@@ -490,6 +491,17 @@ def test_interval_invalid_time_format(invalid_input, error_regex):
     assert re.search(error_regex, exc.value.args[0])
 
 
+@pytest.mark.parametrize("input, result", [("0", 0.0), ("30", 30.0), ("2.5", 2.5), ("1800", 1800.0)])
+def test_nonnegative_seconds(input, result):
+    assert nonnegative_seconds(input) == result
+
+
+@pytest.mark.parametrize("invalid_input", ["-1", "-0.5", "foo", ""])
+def test_nonnegative_seconds_invalid(invalid_input):
+    with pytest.raises(ArgumentTypeError):
+        nonnegative_seconds(invalid_input)
+
+
 @pytest.mark.parametrize(
     "input, result",
     [

From 8c32faec25a3ffa4355567440e2b2406ceca8d28 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Mon, 10 Aug 2026 01:43:55 +0530
Subject: [PATCH 112/144] check --repair: rebuild a corrupt repository index
 from the packs, #10026

A read-only check that finds the repository index corrupt stops and reports it,
as before. With --repair, and only if every pack is intact, the index is now
rebuilt from the packs' object headers and persisted; if any pack is corrupt the
index and the packs are left unchanged and the corruption is reported (salvaging
a corrupt pack's still-intact objects is not implemented yet, refs #8572).

Packs are named and verified by the sha256 of their content, which is
content-addressing rather than a MAC, so this rebuild detects accidental
corruption but not tampering, refs #9901.

On a full check the archives phase runs after the repository phase, so
ArchiveChecker.finish() now persists the chunks index instead of deleting it:
it rebuilds from the packs when repair changed them, else writes out the index
it already holds, then drops the in-memory copy so close() does not overwrite
it. Previously finish() deleted the on-disk index, forcing a slow rebuild on the
next repository access.

check() gains a repo_only argument: a corrupt pack fails a repository-only
repair, but a full check defers the verdict to the archives phase, which can
repair a corrupt pack holding archive/item metadata (or file content with
--verify-data).

The slow rebuild path now shows a progress indicator.
---
 src/borg/archive.py                           | 28 +++++--
 src/borg/archiver/check_cmd.py                | 10 ++-
 src/borg/cache.py                             | 12 ++-
 src/borg/repository.py                        | 73 ++++++++++++++-----
 src/borg/testsuite/archiver/check_cmd_test.py | 30 ++++++++
 src/borg/testsuite/repository_test.py         | 52 +++++++++++++
 6 files changed, 175 insertions(+), 30 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index db7de5cd03..5a95c875a9 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -23,7 +23,7 @@
 
 from . import xattr
 from .chunkers import get_chunker, Chunk, release_chunk_data
-from .cache import ChunkListEntry, build_chunkindex_from_repo, delete_chunkindex_from_repo
+from .cache import ChunkListEntry, build_chunkindex_from_repo, write_chunkindex_to_repo
 from .crypto.key import key_factory, UnsupportedPayloadError
 from .constants import *  # NOQA
 from .crypto.low_level import IntegrityError as IntegrityErrorBase
@@ -1884,6 +1884,9 @@ class ArchiveChecker:
     def __init__(self):
         self.error_found = False
         self.key = None
+        # True once repair drops a defect chunk or writes a new one, i.e. once the chunks index no
+        # longer matches the packs.
+        self.chunks_modified = False
 
     def check(
         self,
@@ -2081,6 +2084,7 @@ def verify_data(self):
                         # keeping the other chunks. update_index=False: finish() rebuilds the index from
                         # the rewritten packs anyway, so a per-chunk full index write would be wasted.
                         self.repository.delete(defect_chunk, update_index=False)
+                        self.chunks_modified = True
                         # drop it from our own index too, so rebuild_archives reports the file it belongs to.
                         del self.chunks[defect_chunk]
                     else:
@@ -2237,6 +2241,7 @@ def add_reference(id_, size, cdata):
                 if self.repair:
                     pack_results = self.repository.put(id_, cdata)
                     self.chunks.update_pack_info(pack_results)
+                    self.chunks_modified = True
 
         def verify_file_chunks(archive_name, item):
             """Verify that all of a file's chunks are present, collecting any missing ones for the report."""
@@ -2450,13 +2455,20 @@ def valid_item(obj):
 
     def finish(self):
         if self.repair:
-            # store packs still buffered from chunks re-added during repair, so their index entries
-            # are set before the index is dropped below and no chunk is left buffered at close().
-            self.repository.flush()
-            # we may have deleted chunks. delete_chunkindex_from_repo() removes the on-disk index and
-            # drops the stale in-memory index, so the next repository access rebuilds it from the repo.
-            logger.info("Deleting chunk indexes in repository - next repository access will cause a rebuild.")
-            delete_chunkindex_from_repo(self.repository)
+            if self.chunks_modified:
+                # the packs changed, so the index no longer matches them: rebuild it from the packs
+                # and persist it. flush first so the rewritten and newly written packs are on the store.
+                self.repository.flush()
+                logger.info("Rebuilding and writing the repository chunks index.")
+                build_chunkindex_from_repo(self.repository, slow_rebuild=True, write_immediately=True)
+            else:
+                # the packs are unchanged, so the index still matches them: persist it as is.
+                logger.info("Writing the rebuilt repository chunks index.")
+                write_chunkindex_to_repo(
+                    self.repository, self.chunks, incremental=False, clear=False, force_write=True, delete_other=True
+                )
+            # drop the in-memory index so close() does not persist it over the index just written.
+            self.repository.invalidate_chunk_index()
             logger.info("Writing Manifest.")
             self.manifest.write()
 
diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index fa31c6a98b..717051fcb1 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -78,7 +78,9 @@ def do_check(self, args, repository):
             # the repository check has finished, which can take hours.
             ArchiveFormatter.validate_format(format)
         if not args.archives_only:
-            if not repository.check(repair=args.repair, max_duration=args.max_duration, max_age=max_age):
+            if not repository.check(
+                repair=args.repair, max_duration=args.max_duration, max_age=max_age, repo_only=args.repo_only
+            ):
                 set_ec(EXIT_WARNING)
             if sig_int:  # repository check interrupted; skip the archive check
                 raise Error("Got Ctrl-C / SIGINT.")
@@ -232,8 +234,10 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
 
         In practice, repair mode hooks into both the repository and archive checks:
 
-        1. When checking the repository's consistency, repair mode removes corrupted
-           objects from the repository after it did a 2nd try to read them correctly.
+        1. When checking the repository's consistency, repair mode rebuilds the repository
+           index from the packs if the index is corrupt, provided every pack is intact. If
+           any pack is corrupt, the index is left as-is and the corruption is reported;
+           salvaging a corrupt pack's still-intact objects is not implemented yet.
 
         2. When checking the consistency and correctness of archives, repair mode might
            remove whole archives from the manifest if their archive metadata chunk is
diff --git a/src/borg/cache.py b/src/borg/cache.py
index e49e3533be..b018879b38 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -30,7 +30,7 @@
 from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list
 from .helpers import format_file_size, safe_encode
 from .helpers import safe_ns
-from .helpers import ProgressIndicatorMessage
+from .helpers import ProgressIndicatorMessage, ProgressIndicatorPercent
 from .helpers import msgpack
 from .helpers.msgpack import int_to_timestamp, timestamp_to_int
 from .item import ChunkListEntry
@@ -938,15 +938,23 @@ def build_chunkindex_from_repo(
     # headers and skipping the (much larger) encrypted payloads. Don't call Repository.list() here:
     # it iterates this same index we are building, so it would recurse. The headers also give each
     # object's real (chunk_id, offset, size), so every object in a pack is indexed individually.
-    for info in repository.store_list("packs"):
+    pack_infos = repository.store_list("packs")
+    pi = ProgressIndicatorPercent(
+        total=len(pack_infos), msg="Rebuilding chunk index %3.0f%%", msgid="cache.build_chunkindex_from_repo"
+    )
+    for info in pack_infos:
         # PackReader uses the store directly, so refresh the lock here; a full rebuild can be slow.
         repository._lock_refresh()
+        pi.show(increase=1)
         pack_id = hex_to_bin(info.name)
         for chunk_id, obj_offset, obj_size in PackReader(repository.store, pack_id).iter_headers():
             num_chunks += 1
             chunks[chunk_id] = ChunkIndexEntry(
                 flags=init_flags, size=0, pack_id=pack_id, obj_offset=obj_offset, obj_size=obj_size
             )
+    if pack_infos:
+        pi.show(current=len(pack_infos))  # finish at 100%
+    pi.finish()
     duration = perf_counter() - t0 or 0.001
     # Chunk IDs in a list are encoded in 34 bytes: 1 byte msgpack header, 1 byte length, 32 ID bytes.
     # Protocol overhead is neglected in this calculation.
diff --git a/src/borg/repository.py b/src/borg/repository.py
index d6a851237f..67349b8531 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -992,7 +992,7 @@ def info(self):
         info = dict(id=self.id, version=self.version)
         return info
 
-    def check(self, repair=False, max_duration=0, max_age=0):
+    def check(self, repair=False, max_duration=0, max_age=0, repo_only=False):
         """Check repository consistency.
 
         packs/ and index/ objects are named by the sha256 of their content, so a pack or index file
@@ -1002,14 +1002,18 @@ def check(self, repair=False, max_duration=0, max_age=0):
         The index is hashed first and the packs only if it is intact. The packs could be hashed even
         with a corrupt index, but a corrupt index already means the user has to repair it, and that
         rebuild re-reads every pack anyway - so a read-only check just stops and reports it instead of
-        continuing. The index is never rebuilt here in any case: reading every pack to do so would be
-        far too slow and expensive for a routine (e.g. cron) check. Salvaging good objects out of
-        corrupt packs and dropping those packs is left to repair, refs #8572. The ids of the packs
-        found corrupt are kept in cache/checked-packs for repair, refs #9696.
+        continuing. A read-only check never rebuilds the index: reading every pack to do so would be
+        far too slow and expensive for a routine (e.g. cron) check. With repair=True and a corrupt
+        index, and if every pack is intact, the index is rebuilt from the packs' object headers and
+        persisted; on a full check the archives phase rebuilds and re-persists it afterwards, see
+        ArchiveChecker.finish. Packs are verified by sha256, which is content-addressing rather than a
+        MAC, so this rebuild detects accidental corruption but not tampering, refs #9901, #10026. If any
+        pack is corrupt the index is left unchanged, refs #8572, #10026. Pack ids found corrupt are kept
+        in cache/checked-packs, refs #9696.
 
         A pack recorded corrupt fails the check, also on a partial run that stops before re-reaching
         it. The record clears at the check that finds the pack intact again or gone (removed by
-        compact, or salvaged and dropped by repair; refs #8572); prune() does this from packs/.
+        compact; TODO: also when repair salvages and drops it, refs #8572); prune() does this from packs/.
 
         It also reports missing packs (refs #9898): pack ids the chunk index references but that are
         absent from packs/. The index is read from its fragments only and its referenced pack ids are
@@ -1022,6 +1026,10 @@ def check(self, repair=False, max_duration=0, max_age=0):
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
         max_age, accepting a future timestamp up to MAX_CLOCK_SKEW (clock skew). Results are recorded
         regardless of max_age.
+
+        repo_only: whether this is a repository-only run. In repair mode it sets the return value for a
+        corrupt pack, which repair does not fix: fail if repo_only, else defer (a full check's archives
+        phase can repair a corrupt pack holding metadata, or file content with --verify-data).
         """
 
         def verify(namespace, name):
@@ -1061,6 +1069,8 @@ def store_list(namespace):
         index_files = index_errors = 0
         pack_files = pack_errors = pack_skipped = 0
         missing_pack_ids = []  # packs referenced by the index but absent from packs/ (refs #9898)
+        index_repaired = False
+        packs_scanned = False
         # index and packs get separate progress indicators, each running from 0% to 100%.
         # the index is checked first and in full, on partial checks too: it is small, and index errors
         # stop the pack check below.
@@ -1082,7 +1092,13 @@ def store_list(namespace):
         if index_infos:
             index_pi.show(current=len(index_infos))  # finish at 100%
         index_pi.finish()
-        if index_errors == 0:
+        if index_errors == 0 or repair:
+            # verify the packs; during repair, rebuild the corrupt index from them afterwards.
+            # --repair forbids --max-duration and --max-age, so the partial and max_age handling in
+            # the loop stays inactive during a repair.
+            packs_scanned = True
+            if index_errors:
+                logger.warning("Repository index is corrupted; rebuilding it from the packs.")
             # packs are the bulk of the work and the part --max-duration spreads over several checks.
             pack_infos = store_list("packs")
             # drop objects whose name is not a valid pack name and count them as errors; the code
@@ -1172,8 +1188,16 @@ def recorded_ts(info):
                 logger.info("Finished checking packs.")
             tracker.prune(present_pack_ids)
             pack_pi.finish()
+            if index_errors and pack_errors == 0:
+                from .cache import build_chunkindex_from_repo
+
+                # rebuild the index from the packs. the exclusive check lock keeps the pack set
+                # fixed, so re-listing packs/ inside build_chunkindex_from_repo matches this
+                # verification. write_immediately persists the index and drops the corrupt fragments.
+                build_chunkindex_from_repo(self, slow_rebuild=True, write_immediately=True)
+                self.invalidate_chunk_index()  # the rebuilt index is persisted; drop the in-memory copy
+                index_repaired = True
         else:
-            # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
         objs_errors = index_errors + pack_errors + len(missing_pack_ids)
         summary = (
@@ -1192,11 +1216,13 @@ def recorded_ts(info):
                 "The chunks stored in these packs are lost. Repairing the index (dropping the "
                 "stale references) is tracked in https://github.com/borgbackup/borg/issues/8572."
             )
-        # corrupt_ids() is every pack recorded corrupt, including from earlier runs. with a corrupt
-        # index the packs were not scanned, so report nothing.
-        corrupt_ids = tracker.corrupt_ids() if index_errors == 0 else []
+        if index_repaired:
+            logger.info("Repository index was corrupted and has been rebuilt from the packs.")
+        # corrupt_ids() includes packs recorded corrupt in earlier runs; report them only when this
+        # run scanned the packs.
+        corrupt_ids = tracker.corrupt_ids() if packs_scanned else []
         if corrupt_ids:
-            # one id per line (the list can be long).
+            # one id per line, the list can be long.
             logger.error(f"Found {len(corrupt_ids)} corrupt pack(s):")
             for pack_id in corrupt_ids:
                 logger.error(f"Corrupt pack: {bin_to_hex(pack_id)}")
@@ -1206,12 +1232,25 @@ def recorded_ts(info):
         done, so_far = ("Interrupted", " so far") if sig_int else ("Finished", "")
         if not problems:
             logger.info(f"{done} {mode} repository check, no problems found{so_far}.")
-        elif repair:
-            logger.error(f"{done} {mode} repository check, errors found{so_far} (repository repair not implemented).")
-        else:
+        elif not repair:
             logger.error(f"{done} {mode} repository check, errors found{so_far}.")
-        # True means the checked objects were clean; --repair returns True so the caller proceeds to fix them.
-        return not problems or repair
+        elif not (pack_errors or corrupt_ids):
+            # only the index was corrupt, and it was rebuilt.
+            logger.info(f"{done} {mode} repository check, repaired{so_far}.")
+        elif repo_only:
+            logger.error(
+                f"{done} {mode} repository check, corrupt pack(s) found{so_far}; repairing a repository "
+                "with corrupt packs is not implemented yet (refs #8572)."
+            )
+        else:
+            # a full check's archives phase reads archive/item metadata (and file content with
+            # --verify-data), so it repairs a corrupt pack holding such objects; warn rather than fail.
+            logger.warning(f"{done} {mode} repository check, corrupt pack(s) found{so_far}.")
+        # in repair mode a corrupt pack fails only a repository-only run; a full check defers to the
+        # archives phase.
+        if repair:
+            return not (repo_only and (pack_errors or corrupt_ids))
+        return not problems
 
     def list(self, limit=None, marker=None):
         """
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index f9b881cc38..f738db6b7b 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -594,6 +594,36 @@ def test_spoofed_manifest(archivers, request):
     cmd(archiver, "check", exit_code=0)
 
 
+def test_check_repair_rebuilds_corrupt_index(archivers, request):
+    # A corrupt index with all packs intact: the default (full) --repair rebuilds the index from the
+    # packs and persists it (via the archives check, see ArchiveChecker.finish), leaving the repository
+    # usable again without a slow rebuild on the next access.
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+    cmd(archiver, "check", exit_code=0)
+    archive, repository = open_archive(archiver.repository_path, "archive1")
+    with repository:
+        assert isinstance(repository, Repository)
+        for info in repository.store_list("index"):  # rot every index fragment
+            name = f"index/{info.name}"
+            data = bytearray(repository.store_load(name))
+            data[0] ^= 0xFF
+            repository.store_store(name, bytes(data))
+    cmd(archiver, "check", exit_code=1)  # read-only check reports the corrupt index
+    output = cmd(archiver, "check", "-v", "--repair", exit_code=0)
+    assert "rebuilt" in output.lower()
+    # item 6: repair persisted a fresh index instead of leaving it for a slow rebuild on the next
+    # access. confirm the on-disk index exists and every fragment is intact.
+    archive, repository = open_archive(archiver.repository_path, "archive1")
+    with repository:
+        index_infos = list(repository.store_list("index"))
+        assert index_infos  # a fresh index was persisted
+        for info in index_infos:  # each fragment's content still matches its sha256 name
+            assert repository.store.hash(f"index/{info.name}") == info.name
+    cmd(archiver, "check", exit_code=0)  # the repository is consistent again
+    assert "archive1" in cmd(archiver, "repo-list")  # and remains usable
+
+
 @pytest.mark.skip(reason="TODO: repair does not yet rewrite store-corrupted packs, refs #8572")
 def test_manifest_rebuild_corrupted_chunk(archivers, request):
     archiver = request.getfixturevalue(archivers)
diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index de21668bef..c9020dd851 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1080,6 +1080,58 @@ def test_check_reports_invalid_pack_name(tmp_path, caplog):
         assert after.table[intact_id].result == 1  # the valid pack was checked
 
 
+def test_check_repair_rebuilds_corrupt_index(tmp_path):
+    # check(repair=True) rebuilds a corrupt index from the packs' object headers.
+    location = os.fspath(tmp_path / "repo")
+    ids = [H(x) for x in range(10)]
+    with Repository(location, exclusive=True, create=True) as repository:
+        for i, cid in enumerate(ids):
+            repository.put(cid, fchunk(bytes([i]) * 20, chunk_id=cid))
+        repository.flush()  # seal the pack(s) and let close() persist the index
+    with reopen(repository) as repository:
+        index_names = [f"index/{info.name}" for info in repository.store_list("index")]
+        assert index_names  # close() persisted at least one index fragment
+        for name in index_names:  # rot every fragment so its content no longer matches its sha256 name
+            data = bytearray(repository.store_load(name))
+            data[0] ^= 0xFF
+            repository.store_store(name, bytes(data))
+        assert repository.check(repair=False) is False  # read-only check reports the corrupt index
+    with reopen(repository) as repository:
+        assert repository.check(repair=True) is True  # repair rebuilds the index from the packs
+    with reopen(repository) as repository:
+        assert repository.check(repair=False) is True  # the rebuilt index passes a read-only check
+        for i, cid in enumerate(ids):
+            assert pdchunk(repository.get(cid)) == bytes([i]) * 20  # every chunk is indexed and resolves
+
+
+def test_check_repair_refuses_when_pack_corrupt(tmp_path):
+    # A repair that finds any corrupt pack leaves the index and the pack untouched (no lossy rebuild,
+    # nothing dropped) and fails on a repository-only run, refs #8572, #10026.
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        repository.put(H(1), fchunk(b"GOOD-CHUNK", chunk_id=H(1)))
+        repository.flush()  # seal a pack holding H(1)
+        repository.put(H(2), fchunk(b"LOST-CHUNK", chunk_id=H(2)))
+        repository.flush()  # seal a separate pack holding H(2)
+    with reopen(repository) as repository:
+        bad_pack_name = "packs/" + bin_to_hex(repository.chunks[H(2)].pack_id)
+        data = bytearray(repository.store_load(bad_pack_name))
+        data[-1] ^= 0xFF  # rot the pack holding H(2): its content no longer matches its sha256 name
+        repository.store_store(bad_pack_name, bytes(data))
+        for info in repository.store_list("index"):  # rot the index so repair takes the rebuild path
+            name = f"index/{info.name}"
+            idata = bytearray(repository.store_load(name))
+            idata[0] ^= 0xFF
+            repository.store_store(name, bytes(idata))
+    with reopen(repository) as repository:
+        # a repository-only repair cannot fix a corrupt pack, so it fails.
+        assert repository.check(repair=True, repo_only=True) is False
+        # the corrupt pack is left in place, not dropped.
+        assert bad_pack_name in [f"packs/{info.name}" for info in repository.store_list("packs")]
+    with reopen(repository) as repository:
+        assert repository.check(repair=False) is False  # index was not rebuilt; still corrupt
+
+
 def test_check_warns_on_invalid_chunk_index(tmp_path, caplog):
     # check warns about an invalid chunk index but does not fail, since the index is not part of
     # the repository's object integrity.

From e1fdba6bcf582fbc39e8927b7a223619901d770c Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Wed, 12 Aug 2026 14:17:30 +0530
Subject: [PATCH 113/144] check --repair: only rebuild the index when every
 pack was verified intact this run

Gate the rebuild on an uninterrupted, full pack scan, flush re-added chunks before
rebuilding in finish(), and report/return honestly when the index stays corrupt.
---
 src/borg/archive.py                           |  6 ++-
 src/borg/repository.py                        | 44 ++++++++++++-------
 src/borg/testsuite/archiver/check_cmd_test.py |  4 +-
 3 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index 5a95c875a9..6af3bb0612 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -2455,10 +2455,12 @@ def valid_item(obj):
 
     def finish(self):
         if self.repair:
+            # flush chunks re-added during repair so their packs are on the store and out of the pack
+            # writer buffer (close() requires an empty buffer, #10055) before we (re)build the index.
+            self.repository.flush()
             if self.chunks_modified:
                 # the packs changed, so the index no longer matches them: rebuild it from the packs
-                # and persist it. flush first so the rewritten and newly written packs are on the store.
-                self.repository.flush()
+                # and persist it.
                 logger.info("Rebuilding and writing the repository chunks index.")
                 build_chunkindex_from_repo(self.repository, slow_rebuild=True, write_immediately=True)
             else:
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 67349b8531..56e597fe0e 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1098,7 +1098,7 @@ def store_list(namespace):
             # the loop stays inactive during a repair.
             packs_scanned = True
             if index_errors:
-                logger.warning("Repository index is corrupted; rebuilding it from the packs.")
+                logger.warning("Repository index is corrupted; verifying all packs before rebuilding it from them.")
             # packs are the bulk of the work and the part --max-duration spreads over several checks.
             pack_infos = store_list("packs")
             # drop objects whose name is not a valid pack name and count them as errors; the code
@@ -1188,12 +1188,15 @@ def recorded_ts(info):
                 logger.info("Finished checking packs.")
             tracker.prune(present_pack_ids)
             pack_pi.finish()
-            if index_errors and pack_errors == 0:
+            # rebuild only if the index was the sole problem and every pack was verified intact this
+            # run: sig_int breaks the loop early, so "no pack errors" must be paired with "all packs
+            # scanned" (pack_files == len(pack_infos)) to not rebuild from unverified packs.
+            if index_errors and pack_errors == 0 and not sig_int and pack_files == len(pack_infos):
                 from .cache import build_chunkindex_from_repo
 
-                # rebuild the index from the packs. the exclusive check lock keeps the pack set
-                # fixed, so re-listing packs/ inside build_chunkindex_from_repo matches this
-                # verification. write_immediately persists the index and drops the corrupt fragments.
+                # the exclusive check lock keeps the pack set fixed, so re-listing packs/ inside
+                # build_chunkindex_from_repo matches this verification. write_immediately persists the
+                # index and drops the corrupt fragments.
                 build_chunkindex_from_repo(self, slow_rebuild=True, write_immediately=True)
                 self.invalidate_chunk_index()  # the rebuilt index is persisted; drop the in-memory copy
                 index_repaired = True
@@ -1234,21 +1237,28 @@ def recorded_ts(info):
             logger.info(f"{done} {mode} repository check, no problems found{so_far}.")
         elif not repair:
             logger.error(f"{done} {mode} repository check, errors found{so_far}.")
-        elif not (pack_errors or corrupt_ids):
-            # only the index was corrupt, and it was rebuilt.
+        elif index_repaired and not (pack_errors or corrupt_ids):
+            # the index was the only problem and it has been rebuilt from the packs.
             logger.info(f"{done} {mode} repository check, repaired{so_far}.")
-        elif repo_only:
-            logger.error(
-                f"{done} {mode} repository check, corrupt pack(s) found{so_far}; repairing a repository "
-                "with corrupt packs is not implemented yet (refs #8572)."
-            )
+        elif pack_errors or corrupt_ids:
+            if repo_only:
+                logger.error(
+                    f"{done} {mode} repository check, corrupt pack(s) found{so_far}; repairing a repository "
+                    "with corrupt packs is not implemented yet (refs #8572)."
+                )
+            else:
+                # a full check's archives phase reads archive/item metadata (and file content with
+                # --verify-data), so it repairs a corrupt pack holding such objects; warn rather than fail.
+                logger.warning(f"{done} {mode} repository check, corrupt pack(s) found{so_far}.")
         else:
-            # a full check's archives phase reads archive/item metadata (and file content with
-            # --verify-data), so it repairs a corrupt pack holding such objects; warn rather than fail.
-            logger.warning(f"{done} {mode} repository check, corrupt pack(s) found{so_far}.")
-        # in repair mode a corrupt pack fails only a repository-only run; a full check defers to the
-        # archives phase.
+            # the index is corrupt but was not rebuilt, e.g. the pack verification was interrupted
+            # before every pack was confirmed intact; the corrupt index is left in place.
+            logger.error(f"{done} {mode} repository check, index still corrupt{so_far}.")
+        # in repair mode a corrupt index left unrebuilt is a failure; a corrupt pack fails only a
+        # repository-only run, while a full check defers it to the archives phase.
         if repair:
+            if index_errors and not index_repaired:
+                return False
             return not (repo_only and (pack_errors or corrupt_ids))
         return not problems
 
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index f738db6b7b..85123a8d0e 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -719,7 +719,7 @@ def test_extra_chunks(archivers, request):
 
 
 def test_repair_finish_flushes_pack_writer(archivers, request):
-    """finish() stores chunks re-added during --repair before it drops the index (#10055).
+    """finish() stores chunks re-added during --repair before it (re)builds the index (#10055).
 
     close() asserts an empty pack writer buffer, so a chunk left buffered by finish() would
     trip it.
@@ -736,6 +736,8 @@ def test_repair_finish_flushes_pack_writer(archivers, request):
         checker.repository = repository
         checker.key = checker.make_key(repository)
         checker.manifest = Manifest.load(repository, (Manifest.Operation.CHECK,), key=checker.key)
+        # re-adding a chunk makes the chunks index no longer match the packs, so finish() rebuilds it.
+        checker.chunks_modified = True
 
         # a chunk re-added during repair, buffered in the pack writer:
         key = b"01234567890123456789012345678901"

From 8b80d148170e3899bb5d25fcf2e5556f7361b3b5 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Thu, 13 Aug 2026 13:23:53 +0530
Subject: [PATCH 114/144] check --repair: scope the corrupt-pack index note to
 the repository check, soften the pre-rebuild warning

---
 src/borg/archiver/check_cmd.py | 5 +++--
 src/borg/repository.py         | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py
index 717051fcb1..d6aedb8517 100644
--- a/src/borg/archiver/check_cmd.py
+++ b/src/borg/archiver/check_cmd.py
@@ -236,8 +236,9 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
 
         1. When checking the repository's consistency, repair mode rebuilds the repository
            index from the packs if the index is corrupt, provided every pack is intact. If
-           any pack is corrupt, the index is left as-is and the corruption is reported;
-           salvaging a corrupt pack's still-intact objects is not implemented yet.
+           any pack is corrupt, the repository check leaves the index and the packs untouched
+           and reports the corruption; salvaging a corrupt pack's still-intact objects is not
+           implemented yet (refs #8572).
 
         2. When checking the consistency and correctness of archives, repair mode might
            remove whole archives from the manifest if their archive metadata chunk is
diff --git a/src/borg/repository.py b/src/borg/repository.py
index 56e597fe0e..354f6943aa 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1098,7 +1098,10 @@ def store_list(namespace):
             # the loop stays inactive during a repair.
             packs_scanned = True
             if index_errors:
-                logger.warning("Repository index is corrupted; verifying all packs before rebuilding it from them.")
+                logger.warning(
+                    "Repository index is corrupted; verifying all packs before deciding whether to "
+                    "rebuild it from them."
+                )
             # packs are the bulk of the work and the part --max-duration spreads over several checks.
             pack_infos = store_list("packs")
             # drop objects whose name is not a valid pack name and count them as errors; the code

From 8640a75b00b4e7ed5f75a9ff80f303ba13e3cce9 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 13 Aug 2026 11:46:29 +0000
Subject: [PATCH 115/144] create --encryption: new none-* and authenticated-*
 modes, #9104

The old "none" and "authenticated" modes had a no-op repo object envelope: it was
just the key type byte followed by the payload, and it ignored the aad the
RepoObj layer passes. Only the chunk id over the plaintext was ever checked, so
a repo object's metadata and its object header were not verified at all - and
the metadata selects the decompressor and the payload size, i.e. it was used
before anything about the object had been checked.

The new modes now use a tagged envelope instead:

    TYPE(1) + reserved(1) + tag(32) + payload

The tag covers the envelope header, the aad (object header, chunk id and the
meta/data slot tag) and the payload, so tampering with any of them, swapping
the meta and the data slot or taking an object slice from a different object
fails verification - and it does so before the metadata is used for anything.
This also makes the existing header-aad and pack-header binding effective for
these modes, which so far only worked for the AEAD modes.

The tag is deterministic: a MAC needs no nonce, so there is no session state to
manage, and repositories with the same key material store byte-identical
objects for identical input (which allows deduplicating them on the filesystem
level).

  - authenticated-sha256 / authenticated-blake3 (0x60 / 0x70): the tag is a MAC
    (HMAC-SHA256 resp. keyed BLAKE3), so these modes now detect malicious
    tampering with metadata and object header, not just with the chunk content.
    The MAC key is derived from crypt_key, deliberately not from id_key: chunk
    ids are public and related repositories share the id key, which must not
    let them forge each other's objects. --copy-crypt-key opts into sharing it.
  - none-sha256 / none-blake3 (0x80 / 0x90): there is no key, so the tag is an
    unkeyed checksum. It detects accidental corruption (and a ranged read that
    returned the wrong bytes), but anybody can recompute it - these modes make
    no authenticity claim, so their chunk id check stays mandatory.

The unencrypted modes name their id hash now, because the hash is what protects
the data there: bare "none"/"authenticated" are rejected and --id-hash only
applies to the encrypted modes.

The borg 2 beta formats 0x02, 0x07 and 0x50 are dropped; 0x02/0x06/0x07 live on
as read-only legacy classes so "borg transfer --from-borg1" keeps working, and
key_factory refuses them for borg 2 repositories.

Update docs about new modes.

Also correct the packs docs, which said the header aad binding applies to the
AEAD modes only - it applies to all borg 2 modes now.
---
 docs/changes.rst                              |  25 ++
 docs/internals/data-structures.rst            |  56 ++-
 docs/internals/packs.rst                      |  14 +-
 docs/internals/security.rst                   |  62 ++-
 src/borg/archiver/help_cmd.py                 |   8 +-
 src/borg/archiver/repo_create_cmd.py          |  57 +--
 src/borg/archiver/repo_info_cmd.py            |  12 +-
 src/borg/constants.py                         |  10 +-
 src/borg/crypto/key.py                        | 384 ++++++++++++++----
 src/borg/legacy/crypto/key.py                 | 103 ++++-
 src/borg/repoobj.py                           |  30 +-
 src/borg/security.py                          |   5 +-
 src/borg/testsuite/archive_test.py            |  14 +-
 src/borg/testsuite/archiver/check_cmd_test.py |   4 +-
 src/borg/testsuite/archiver/checks_test.py    |  10 +-
 .../testsuite/archiver/create_cmd_test.py     |   6 +-
 src/borg/testsuite/archiver/disk_full_test.py |   2 +-
 .../testsuite/archiver/extract_cmd_test.py    |  12 +-
 src/borg/testsuite/archiver/key_cmds_test.py  |  32 +-
 src/borg/testsuite/archiver/lock_cmds_test.py |   2 +-
 .../testsuite/archiver/mount_cmds_test.py     |   2 +-
 .../testsuite/archiver/recreate_cmd_test.py   |   2 +-
 .../archiver/repo_create_cmd_test.py          |  56 ++-
 .../testsuite/archiver/return_codes_test.py   |   2 +-
 src/borg/testsuite/archiver/tar_cmds_test.py  |  26 +-
 src/borg/testsuite/benchmark_test.py          |   2 +-
 src/borg/testsuite/cockpit_test.py            |   2 +-
 src/borg/testsuite/crypto/crypto_test.py      |   8 +-
 src/borg/testsuite/crypto/key_test.py         | 226 ++++++++++-
 src/borg/testsuite/legacy_archives_test.py    |   2 +-
 src/borg/testsuite/repoobj_test.py            | 145 +++++--
 31 files changed, 1075 insertions(+), 246 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 0ac7066af3..b74f07c795 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -167,6 +167,31 @@ Version 2.0.0b23 (not released yet)
 
 New features:
 
+- crypto: protect metadata and object header in the modes that do not encrypt, #9104.
+
+  The "none" and "authenticated" modes had a no-op repo object envelope: only the chunk
+  id over the plaintext was checked, so a repo object's metadata (which selects the
+  decompressor!) and its object header were not verified at all. They now use a tagged
+  envelope: every object slot carries a 32 byte tag over the payload, the object header,
+  the chunk id and the meta/data slot, verified before the payload is used.
+
+  - "authenticated-sha256" / "authenticated-blake3": the tag is a MAC (HMAC-SHA256 resp.
+    keyed BLAKE3, key derived from crypt_key), thus these modes now detect tampering with
+    metadata and object header, not just with the chunk content.
+  - "none-sha256" / "none-blake3": the tag is an unkeyed checksum, which detects accidental
+    corruption. It is no authentication - these modes have no key and make no such claim.
+  - The tag is deterministic (no nonce/session), so repositories with the same key material
+    store byte-identical objects for identical input.
+  - The unencrypted modes are named "<mode>-<hash>" now, because the hash *is* what protects
+    the data there: "none-sha256", "none-blake3", "authenticated-sha256",
+    "authenticated-blake3". Bare "--encryption none" / "--encryption authenticated" are
+    rejected, and "--id-hash" only applies to the encrypted modes now.
+  - "none-blake3" is new: the unencrypted mode was limited to sha256 before.
+
+  Breaking: borg 2 beta repositories using the old "none" or "authenticated" formats are not
+  supported any more - create a new repository, or transfer the archives with borg 2.0.0b23
+  or older first. Reading borg 1.x "none"/"authenticated" repositories for
+  "borg transfer --from-borg1" is not affected.
 - faster create:
 
   - use multi-threaded zstd compression for big chunks, #9961
diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index 4823404b5e..a90be6672d 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -920,6 +920,54 @@ need any special handling when reading, because the session id is part of every
 header. Because the advantages of the individual session keys just add up, frequent
 session key changes also keep the total advantage low over the lifetime of a borg key.
 
+.. _tagged_envelope:
+
+Modes without encryption
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``authenticated-*`` and ``none-*`` modes do not encrypt: the payload of a repository
+object slot (the compressed chunk data resp. the packed metadata, see `Repository objects`_)
+is stored as-is. Every slot still carries a 32 byte tag::
+
+    TYPE(1) + reserved(1) + tag(32) + payload
+
+``TYPE`` is the key type byte (which identifies the mode, see ``KeyType``), ``reserved`` is
+zero. The tag is computed over the envelope header, the AAD and the payload::
+
+    aad_full = aad + chunk_id
+    tag = MAC(tag_key, TYPE || reserved || len16_be(aad_full) || aad_full || payload)
+
+``aad`` is what ``RepoObj`` authenticates alongside the payload: the object header prefix
+(magic, format version, chunk id) and the slot tag (``M`` for meta, ``D`` for data), see
+:ref:`pack-format`. Consequently, the tag detects modification of the payload, of the
+metadata, of the object header, a swap of the meta and the data slot, and an object slice
+taken from a different object. The length prefix keeps the boundary between the AAD and the
+payload unambiguous.
+
+There is no nonce, no session and no other state: the tag is deterministic. Two repositories
+with the same key material therefore store byte-identical objects for identical input, which
+allows deduplicating them on the filesystem level (e.g. with CoW/dedup tools).
+
+The modes differ in the tag algorithm and in whether they have a key at all:
+
+- ``authenticated-sha256`` / ``authenticated-blake3``: the tag is a **MAC** (HMAC-SHA256 resp.
+  keyed BLAKE3), so only somebody who has the borg key can compute it - this detects malicious
+  tampering, not just accidental corruption. The MAC key is derived from ``crypt_key``::
+
+      tag_key = sha256(crypt_key + b"borg-repoobj-mac-hmac-sha256")[:32]   # authenticated-sha256
+      tag_key = sha256(crypt_key + b"borg-repoobj-mac-blake3")[:32]        # authenticated-blake3
+
+  It is deliberately not derived from ``id_key``: chunk ids are public, and related repositories
+  share the id key (see ``borg repo-create --other-repo``), which must not enable them to forge
+  each other's objects. ``--copy-crypt-key`` shares ``crypt_key`` and thus opts into producing
+  byte-identical objects across the related repositories.
+- ``none-sha256`` / ``none-blake3``: there is no key at all, so the tag is an **unkeyed** hash
+  (plain SHA-256 resp. BLAKE3 over the same input), i.e. a checksum. It detects accidental
+  corruption and reads that returned the wrong bytes, but anybody who modifies an object can
+  recompute it - it is no protection against malicious tampering. For the same reason, the chunk
+  ids of these modes are unkeyed hashes of the plaintext, which makes all repositories of such a
+  mode dedup identically.
+
 Legacy modes
 ~~~~~~~~~~~~
 
@@ -929,8 +977,12 @@ Old repositories (which used AES-CTR mode) are supported read-only to be able to
 AES-CTR mode is not supported for new repositories and the related code will be
 removed in a future release.
 
-Both modes
-~~~~~~~~~~
+The same applies to the borg 1.x ``none`` and ``authenticated`` modes: their envelope is just
+the type byte followed by the payload, so nothing about an object is verified except the chunk
+id over the plaintext. They were replaced by the tagged modes described above.
+
+All modes
+~~~~~~~~~
 
 Encryption keys (and other secrets) are kept either in the keys directory on
 the client ('keyfile' mode) or under the keys/ namespace in the repository
diff --git a/docs/internals/packs.rst b/docs/internals/packs.rst
index 638f772ce4..0f8d6acec9 100644
--- a/docs/internals/packs.rst
+++ b/docs/internals/packs.rst
@@ -54,12 +54,14 @@ The fixed part of each blob header is 49 bytes (``REPOOBJ_HEADER_SIZE``):
 ``REPOOBJ_HEADER_SIZE = len(OBJ_MAGIC) + 1 + 32 + 4 + 4 = 49``
 
 Format version ``0x02`` (``OBJ_VERSION_HEADER_AAD``) binds the header's first 41 bytes (``OBJ_MAGIC``
-+ version + ``chunk_id`` -- ``REPOOBJ_HEADER_AAD_SIZE``) into the AEAD authentication of
++ version + ``chunk_id`` -- ``REPOOBJ_HEADER_AAD_SIZE``) into the authentication of
 ``encrypted_meta`` and ``encrypted_data`` as additional authenticated data (AAD: data that is
-authenticated together with the ciphertext, but not itself encrypted). This applies to the AEAD
-encryption modes (AES-256-OCB, ChaCha20-Poly1305). ``meta_size`` and ``data_size`` are excluded from
-the AAD; tampering with either still fails authentication, because it changes the length of the
-ciphertext slice being decrypted. A forged ``chunk_id``, version, or magic byte therefore fails AEAD
+authenticated together with the ciphertext, but not itself encrypted). This applies to all borg 2
+modes: the AEAD encryption modes (AES-256-OCB, ChaCha20-Poly1305) authenticate it with their AEAD
+tag, the ``authenticated-*`` modes with their MAC and the ``none-*`` modes with their (unkeyed)
+checksum, see :ref:`tagged_envelope`. ``meta_size`` and ``data_size`` are excluded from
+the AAD; tampering with either still fails the check, because it changes the length of the
+slice being read. A forged ``chunk_id``, version, or magic byte therefore fails
 authentication in ``RepoObj.parse()``/``parse_meta()``.
 
 ``encrypted_meta`` and ``encrypted_data`` each add a one-byte slot tag on top of the shared header
@@ -82,7 +84,7 @@ decrypting, so it does not check header AAD authentication.
 
     The fixed 49-byte blob header. ``meta_size`` and ``data_size`` drive
     traversal; integrity comes from the content-addressed pack name and the
-    per-blob AEAD, which authenticates magic/version/chunk_id as additional
+    per-blob tag, which authenticates magic/version/chunk_id as additional
     authenticated data.
 
 A reader locates the next blob by advancing::
diff --git a/docs/internals/security.rst b/docs/internals/security.rst
index 079ca2be72..ad09641c91 100644
--- a/docs/internals/security.rst
+++ b/docs/internals/security.rst
@@ -75,10 +75,21 @@ object's metadata (like e.g.: manifest vs. archive vs. user file content data).
 When loading data from the repo, borg verifies that the type of object it got
 matches the type it wanted. borg 2 does not use TAMs any more.
 
-As both the object's metadata and data are AEAD encrypted and also bound to
+As both the object's metadata and data are authenticated and also bound to
 the object ID (via giving the ID as AAD), there is no way an attacker (without
 access to the borg key) could change the type of the object or move content
-to a different object ID.
+to a different object ID. This holds for the AEAD encryption modes (where the
+AEAD tag authenticates them) as well as for the ``authenticated-*`` modes
+(where a MAC does, see :ref:`tagged_envelope`).
+
+It does **not** hold for the ``none-*`` modes: they have no key, so their objects
+carry an unkeyed checksum rather than a MAC, and an attacker who modifies an
+object can simply recompute it. What still constrains an attacker there is the
+object ID being the (unkeyed) hash of the plaintext: the content of an existing
+object can not be replaced without the ID no longer matching. But the object's
+metadata, the archives list and the manifest are not anchored to anything secret,
+so a ``none-*`` repository provides no tamper protection - only detection of
+accidental corruption.
 
 This effectively 'anchors' each archive to the key, which is controlled by the
 client, thereby anchoring the DAG starting from the archives list entry,
@@ -201,10 +212,50 @@ Notable:
   recommended periodic audit for this).
 
 
+Authenticated modes
+~~~~~~~~~~~~~~~~~~~
+
+Modes: ``--encryption authenticated-(sha256|blake3)``
+
+Supported: borg 2.0+
+
+These modes do not encrypt: everything in the repository is readable by anybody who
+can read the repository. They do authenticate, though - like an AEAD mode minus the
+data encryption:
+
+- Every repository object slot (metadata and data) carries a MAC over the payload, the
+  object header and the slot, see :ref:`tagged_envelope`. Reading verifies it before the
+  payload is used for anything (in particular before decompressing it), so tampering with
+  any of them is detected, whether accidental or malicious.
+- The chunk IDs are MACs over the plaintext, as in the encrypted modes.
+- The MAC key lives in a borg key (repokey or keyfile, ``--key-location``), protected by a
+  passphrase like the keys of the encrypted modes. An attacker without that key can neither
+  forge objects nor chunk IDs.
+
+Different from the AEAD modes, the MAC is deterministic (a MAC needs no nonce): there is no
+session key, no IV and thus no usage limit to observe, and identical input produces identical
+objects.
+
+Unencrypted modes
+~~~~~~~~~~~~~~~~~
+
+Modes: ``--encryption none-(sha256|blake3)``
+
+Supported: borg 2.0+
+
+These modes have no key at all: they neither encrypt nor authenticate. Every repository
+object slot carries an *unkeyed* checksum (see :ref:`tagged_envelope`), which detects
+accidental corruption - bad storage hardware, a truncated write, a read that returned the
+wrong bytes - before the data is used. It is not a protection against an attacker: whoever
+modifies an object can recompute the checksum, and the chunk IDs are unkeyed hashes as well.
+
+You are advised not to use these modes. Use ``authenticated-*`` instead if you do not want
+your data encrypted but do want to detect tampering; it is the same thing plus a key.
+
 Legacy modes
 ~~~~~~~~~~~~
 
-Modes: ``--encryption (repokey|keyfile)[-blake2]``
+Modes: ``--encryption (repokey|keyfile)[-blake2]``, ``--encryption (none|authenticated)``
 
 Supported: borg < 2.0
 
@@ -213,6 +264,11 @@ derived via HMAC-SHA256 or (in the ``-blake2`` variants) Blake2b. ``blake2b`` is
 only used by these legacy modes; new repositories use ``sha256`` or ``blake3``
 (see above).
 
+The borg 1.x ``none`` and ``authenticated`` modes belong here, too: their repository
+objects have no tag at all, so nothing about an object is verified except the chunk ID
+over the plaintext - not even the object's metadata. They were replaced by the modes
+described above, which cover metadata and object header as well.
+
 borg 2.0 does not support creating new repos using these modes,
 but ``borg transfer`` can still read such existing repos.
 
diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py
index 6b8f76831c..1b1ac4d564 100644
--- a/src/borg/archiver/help_cmd.py
+++ b/src/borg/archiver/help_cmd.py
@@ -846,10 +846,16 @@ class HelpMixIn:
                     in WSL1 (Windows Subsystem for Linux 1).
 
                 authenticated_no_key
-                    Work around a lost passphrase or key for an ``authenticated`` mode repository
+                    Work around a lost passphrase or key for an ``authenticated-*`` mode repository
                     (these are only authenticated, but not encrypted).
                     If the key is missing in the repository config, add ``key = anything`` there.
 
+                    Without the key, borg can not verify anything that needs it: neither the
+                    authentication tag of the repository objects nor the chunk ids. It therefore
+                    reads the repository **unverified** - a corrupted or tampered repository will
+                    not be detected. (This only concerns the ``authenticated-*`` modes; the
+                    ``none-*`` modes need no key and keep verifying their checksums.)
+
                     This workaround is **only** for emergencies and **only** to extract data
                     from an affected repository (read-only access)::
 
diff --git a/src/borg/archiver/repo_create_cmd.py b/src/borg/archiver/repo_create_cmd.py
index f36f5064c1..7f5e8f4dc8 100644
--- a/src/borg/archiver/repo_create_cmd.py
+++ b/src/borg/archiver/repo_create_cmd.py
@@ -32,7 +32,7 @@ def do_repo_create(self, args, repository, *, other_repository=None, other_manif
         manifest.write()
         with Cache(repository, manifest, warn_if_unencrypted=False):
             pass
-        if key.ENC_NAME != "none":  # any key-bearing suite (everything except plaintext "none")
+        if key.has_secret_key:  # any key-bearing suite (everything except the "none-*" modes)
             logger.warning(
                 "\n"
                 "IMPORTANT: you will need both KEY AND PASSPHRASE to access this repository!\n"
@@ -125,21 +125,23 @@ def build_parser_repo_create(self, subparsers, common_parser, mid_common_parser)
         Depending on your hardware, hashing and crypto performance may vary widely.
         The easiest way to find out what is fastest is to run ``borg benchmark cpu``.
 
-        A crypto suite is selected by three orthogonal options:
-
-        ``--encryption`` (**required**) selects the cipher / authenticated-encryption algorithm:
+        ``--encryption`` (**required**) selects the mode:
 
         - ``aes256-ocb``: AES256 in OCB mode (encryption + authentication).
         - ``chacha20-poly1305``: ChaCha20 + Poly1305 (encryption + authentication).
-        - ``authenticated``: no encryption, but still authenticates your data (tamper detection).
-        - ``none``: no encryption and no authentication (see the warning below).
+        - ``authenticated-sha256`` / ``authenticated-blake3``: no encryption, but authentication
+          (tamper detection) using HMAC-SHA-256 resp. keyed BLAKE3.
+        - ``none-sha256`` / ``none-blake3``: neither encryption nor authentication, only
+          SHA-256 resp. BLAKE3 checksums (see below).
 
-        ``--id-hash`` selects the id hash function (used for chunk ids and authentication):
+        ``--id-hash`` selects the id hash function of the **encrypted** modes:
 
-        - ``sha256`` (default): HMAC-SHA-256 (or plain SHA-256 for the ``none`` encryption).
+        - ``sha256`` (default): HMAC-SHA-256.
         - ``blake3``: BLAKE3. Often faster on CPUs without SHA hardware acceleration.
 
-        The ``none`` encryption has no key, so it only supports the ``sha256`` id hash.
+        For the modes that do not encrypt, the hash is not just used for the chunk ids, it also is
+        what protects your data - therefore it is part of the mode name there and ``--id-hash``
+        does not apply to them.
 
         ``--key-location`` selects where the key is stored (orthogonal to the crypto suite):
 
@@ -149,20 +151,23 @@ def build_parser_repo_create(self, subparsers, common_parser, mid_common_parser)
           this if you want "passphrase and having-the-key" security.
 
         You can move the key between these locations later with ``borg key change-location``.
-        This also applies to the ``authenticated`` encryption: it does not encrypt your data, but it
-        still has a key (used for the id hash and authentication), so ``--key-location`` selects
-        where that key is stored, just like for the encrypted suites.
-        ``--key-location`` is only ignored for the ``none`` encryption, which has no key at all.
-
-        `none` encryption uses no encryption and no authentication. You are advised NOT to use this
-        as it would expose you to a Denial-of-Service risk (due to how the :ref:`internals_hashindex`
-        works) and other issues (confidentiality, tampering, ...) in case of malicious activity
-        in the repository.
+        This also applies to the ``authenticated-*`` modes: they do not encrypt your data, but they
+        still have a key (used for the id hash and the authentication), so ``--key-location``
+        selects where that key is stored, just like for the encrypted modes.
+        ``--key-location`` is only ignored for the ``none-*`` modes, which have no key at all.
+
+        The ``none-*`` modes use neither encryption nor authentication: everything in the
+        repository is readable by anybody, and while every repository object carries a checksum
+        (which detects accidental corruption, e.g. bad storage hardware), anybody who modifies an
+        object can just recompute that checksum. You are advised NOT to use these modes: in case
+        of malicious activity in the repository, they expose you to a Denial-of-Service risk (due
+        to how the :ref:`internals_hashindex` works) and other issues (confidentiality,
+        tampering, ...).
 
         If you do **not** want to encrypt the contents of your backups, but still want to detect
-        malicious tampering, use ``--encryption authenticated``. It is like an encrypted suite
-        minus the data encryption.
-        To normally work with ``authenticated`` repositories, you will need the passphrase, but
+        malicious tampering, use ``--encryption authenticated-sha256`` (or ``-blake3``). These
+        modes are like an encrypted mode minus the data encryption.
+        To normally work with ``authenticated-*`` repositories, you will need the passphrase, but
         there is an emergency workaround; see ``BORG_WORKAROUNDS=authenticated_no_key`` docs.
 
         Creating a related repository
@@ -213,8 +218,8 @@ def build_parser_repo_create(self, subparsers, common_parser, mid_common_parser)
             required=True,
             choices=encryption_argument_names(),
             action=Highlander,
-            help="select cipher / AE algorithm: 'none', 'authenticated', 'aes256-ocb' or "
-            "'chacha20-poly1305' **(required)**",
+            help="select the mode: 'aes256-ocb', 'chacha20-poly1305', 'authenticated-sha256', "
+            "'authenticated-blake3', 'none-sha256' or 'none-blake3' **(required)**",
         )
         subparser.add_argument(
             "-i",
@@ -222,10 +227,10 @@ def build_parser_repo_create(self, subparsers, common_parser, mid_common_parser)
             metavar="HASH",
             dest="id_hash",
             choices=id_hash_argument_names(),
-            default="sha256",
+            default=None,  # None: not given. Do not default to sha256 here, see key_creator.
             action=Highlander,
-            help="select the id hash function: 'sha256' (default) or 'blake3'. "
-            "The 'none' encryption only supports 'sha256'.",
+            help="select the id hash function of the encrypted modes: 'sha256' or 'blake3'. "
+            "The 'none-*' and 'authenticated-*' modes name their hash themselves.",
         )
         subparser.add_argument(
             "--key-location",
diff --git a/src/borg/archiver/repo_info_cmd.py b/src/borg/archiver/repo_info_cmd.py
index f5dd417a59..132eceff5e 100644
--- a/src/borg/archiver/repo_info_cmd.py
+++ b/src/borg/archiver/repo_info_cmd.py
@@ -26,12 +26,12 @@ def do_repo_info(self, args, repository, manifest, cache):
             # the two dimensions: cipher / AE algorithm (ENC_NAME) and id hash function (IDHASH_NAME).
             storage = getattr(key, "storage", None)
             mode = {KeyBlobStorage.KEYFILE: "keyfile", KeyBlobStorage.REPO: "repokey"}.get(storage)
-            suite = "%s, %s" % (key.ENC_NAME, key.IDHASH_NAME)
-            if key.ENC_NAME in ("none", "authenticated"):
-                # the "none" and "authenticated" encryptions do not encrypt data; "authenticated"
-                # (unlike "none"/plaintext) still has a key stored as a keyfile or repokey, so show
-                # that location when there is one.
-                encryption += "No (%s, %s)" % (mode, suite) if mode else "No"
+            # the "none-*" and "authenticated-*" modes name their id hash in ENC_NAME already.
+            suite = key.ENC_NAME if key.IDHASH_IN_ENC_NAME else "%s, %s" % (key.ENC_NAME, key.IDHASH_NAME)
+            if not key.encrypts:
+                # these modes do not encrypt data; the "authenticated-*" ones (unlike "none-*")
+                # still have a key stored as a keyfile or repokey, so show that location if there is one.
+                encryption += "No (%s, %s)" % (mode, suite) if mode else "No (%s)" % suite
             else:
                 encryption += "Yes (%s, %s)" % (mode, suite) if mode else "Yes (%s)" % suite
             if storage == KeyBlobStorage.KEYFILE:
diff --git a/src/borg/constants.py b/src/borg/constants.py
index 5f6ff8effa..6103eb630e 100644
--- a/src/borg/constants.py
+++ b/src/borg/constants.py
@@ -254,7 +254,15 @@ class KeyType:
     CHPO = 0x20
     BLAKE3AESOCB = 0x30
     BLAKE3CHPO = 0x40
-    BLAKE3AUTHENTICATED = 0x50
+    # 0x50 was a borg2 beta format ("authenticated" with blake3 ids and a no-op envelope). It was
+    # dropped before the 2.0 release, see #9104 - the byte stays reserved so it is not reused.
+    DROPPED_BLAKE3AUTHENTICATED = 0x50
+    # the "authenticated-*" and "none-*" modes: not encrypted, but every repo object slot carries
+    # a tag (a MAC resp. an unkeyed checksum), see MACKeyBase.
+    SHA256AUTHENTICATED = 0x60
+    BLAKE3AUTHENTICATED = 0x70
+    SHA256NONE = 0x80
+    BLAKE3NONE = 0x90
 
 
 CACHE_TAG_NAME = "CACHEDIR.TAG"
diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py
index e635fbfe81..7c805e27eb 100644
--- a/src/borg/crypto/key.py
+++ b/src/borg/crypto/key.py
@@ -5,7 +5,7 @@
 from hashlib import sha256
 from math import ceil
 from pathlib import Path
-from typing import Literal, ClassVar, Optional
+from typing import Any, Literal, ClassVar, Optional
 from collections.abc import Callable
 
 from ..logger import create_logger
@@ -27,7 +27,7 @@
 from ..item import Key, EncryptedKey
 from ..manifest import Manifest
 from ..platform import SaveFile
-from ..repoobj import RepoObj
+from ..repoobj import RepoObj, RepoObj1
 
 
 from .low_level import bytes_to_int, num_cipher_blocks, hmac_sha256
@@ -170,18 +170,27 @@ class UnsupportedKeyFormatError(Error):
 
 
 def key_creator(repository, args, *, other_key=None):
-    # the crypto suite is selected by two orthogonal dimensions: the cipher / AE algorithm
-    # (--encryption) and the id hash function (--id-hash). id-hash is always significant, so e.g.
-    # "--encryption none --id-hash blake3" finds no match and is rejected (none only supports sha256).
+    # For the encrypted modes, the crypto suite is selected by two orthogonal dimensions: the
+    # cipher / AE algorithm ("--encryption") and the id hash function ("--id-hash", defaulting to
+    # sha256). For the modes that do not encrypt, the hash is not a tuning knob but *is* the
+    # mechanism (it authenticates resp. checksums the data), so it is part of the mode name:
+    # "none-sha256", "none-blake3", "authenticated-sha256", "authenticated-blake3". Giving
+    # "--id-hash" in addition to one of those names is only accepted if both agree.
     enc = args.encryption
-    id_hash = getattr(args, "id_hash", "sha256")
+    id_hash = getattr(args, "id_hash", None)  # None: not given, see the --id-hash argparse default
     for key in AVAILABLE_KEY_TYPES:
-        if key.ENC_NAME == enc and key.IDHASH_NAME == id_hash:
+        if key.ENC_NAME != enc:
+            continue
+        if key.IDHASH_IN_ENC_NAME:
+            if id_hash is not None and id_hash != key.IDHASH_NAME:
+                raise Error(
+                    f'The "{enc}" encryption mode always uses the "{key.IDHASH_NAME}" id-hash, '
+                    f'thus --id-hash "{id_hash}" can not be used with it.'
+                )
             return key.create(repository, args, other_key=other_key)
-    raise Error(
-        f'Unsupported --encryption "{enc}" / --id-hash "{id_hash}" combination '
-        f'(the "none" encryption only supports the "sha256" id-hash).'
-    )
+        if key.IDHASH_NAME == (id_hash or "sha256"):
+            return key.create(repository, args, other_key=other_key)
+    raise Error(f'Unsupported --encryption "{enc}" / --id-hash "{id_hash}" combination.')
 
 
 def encryption_argument_names():
@@ -217,7 +226,14 @@ def identify_key(manifest_data):
 def key_factory(repository, manifest_chunk, *, other=False, ro_cls=RepoObj):
     manifest_data = ro_cls.extract_crypted_data(manifest_chunk)
     assert manifest_data, "manifest data must not be zero bytes long"
-    return identify_key(manifest_data).detect(repository, manifest_data, other=other)
+    key_cls = identify_key(manifest_data)
+    if key_cls in LEGACY_KEY_TYPES and ro_cls is not RepoObj1:
+        # A borg 2 repository using a borg 1.x key type: that can only be a repository created by
+        # a borg 2 beta in the old "none" or "authenticated" mode, which have been replaced by the
+        # tagged envelope modes (see MACKeyBase). The legacy key classes only exist to read borg
+        # 1.x repositories (ro_cls is RepoObj1 then), e.g. for "borg transfer --from-borg1".
+        raise UnsupportedPayloadError(manifest_data[0])
+    return key_cls.detect(repository, manifest_data, other=other)
 
 
 def uses_same_chunker_secret(other_key, key):
@@ -230,12 +246,13 @@ def uses_same_chunker_secret(other_key, key):
 def uses_same_id_hash(other_key, key):
     """other_key -> key upgrade: is the id hash the same?"""
     # avoid breaking the deduplication by changing the id hash
-    old_sha256_ids = (PlaintextKey,)
-    new_sha256_ids = (PlaintextKey,)
-    old_hmac_sha256_ids = (AESCTRKey, AuthenticatedKey)
+    old_sha256_ids = (LegacyPlaintextKey,)  # unkeyed sha256 over the plaintext
+    new_sha256_ids = (ChecksumKey,)
+    old_hmac_sha256_ids = (AESCTRKey, LegacyAuthenticatedKey)
     new_hmac_sha256_ids = (AESOCBKey, CHPOKey, AuthenticatedKey)
     # note: we do not support blake2b for new repos, see #8867
     new_blake3_ids = (Blake3AESOCBKey, Blake3CHPOKey, Blake3AuthenticatedKey)
+    new_unkeyed_blake3_ids = (Blake3ChecksumKey,)  # unkeyed blake3 over the plaintext
     same_ids = (
         isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids)
         and isinstance(key, new_hmac_sha256_ids)
@@ -243,6 +260,8 @@ def uses_same_id_hash(other_key, key):
         and isinstance(key, new_blake3_ids)
         or isinstance(other_key, old_sha256_ids + new_sha256_ids)
         and isinstance(key, new_sha256_ids)
+        or isinstance(other_key, new_unkeyed_blake3_ids)
+        and isinstance(key, new_unkeyed_blake3_ids)
     )
     return same_ids
 
@@ -260,11 +279,21 @@ class KeyBase:
     # None means "not creatable this way" (e.g. legacy read-only classes).
     ENC_NAME: ClassVar[str] = None  # override in creatable subclasses
     IDHASH_NAME: ClassVar[str] = None  # override in creatable subclasses (or via id-hash mix-in)
+    # Is the id hash part of the ENC_NAME (e.g. "none-blake3") instead of being selectable via
+    # "--id-hash"? True for the modes that do not encrypt: there, the hash is not a tuning knob,
+    # it is what protects the data, so it belongs to the mode. See key_creator.
+    IDHASH_IN_ENC_NAME: ClassVar[bool] = False
 
     # Storage type (no key blob storage / keyfile / repo). This is only a default seed for the
     # per-instance self.storage; keyfile vs repokey is a property of an individual key, not the class.
     STORAGE: ClassVar[str] = KeyBlobStorage.NO_STORAGE
 
+    # per-instance key location, set in __init__ (see also FlexiKey, which shares these with us
+    # in the classes inheriting from both): declared here so that a subclass in another module
+    # does not have to infer their types across the module import cycle.
+    storage: str  # where this key is/will be stored, one of the KeyBlobStorage values
+    target: Any  # key location: keyfile path or repository object
+
     # Whether a key of this class may be stored as a keyfile or as a repokey (configurable at
     # repo creation via --key-location and changeable later via "borg key change-location").
     LOCATION_CONFIGURABLE = False
@@ -287,6 +316,18 @@ class KeyBase:
     # independently of assert_id() - see AEADKeyBase.assert_id for what assert_id adds on top of that.
     id_check_is_authentication: ClassVar[bool] = True
 
+    # Does this mode encrypt the data at all? False for the modes that only tag it ("none-*",
+    # "authenticated-*" and their borg 1.x predecessors). Different from logically_encrypted,
+    # which is a per-instance property (an encrypted repo with an empty passphrase is not
+    # "logically" encrypted, but it does encrypt).
+    encrypts: ClassVar[bool] = True
+
+    # Does this key class have secret key material?
+    # False for the "none-*" modes: their chunk ids and their envelope checksums are unkeyed, thus
+    # they work without any key - and the authenticated_no_key workaround (which fakes the key
+    # material of a key it can not unlock) must not switch off checks that need no key at all.
+    has_secret_key: ClassVar[bool] = True
+
     # Whether this *particular instance* is encrypted from a practical point of view,
     # i.e. when it's using encryption with a empty passphrase, then
     # that may be *technically* called encryption, but for all intents and purposes
@@ -367,38 +408,6 @@ def unpack_archive(self, data):
         return unpacked
 
 
-class PlaintextKey(KeyBase):
-    TYPE = KeyType.PLAINTEXT
-    TYPES_ACCEPTABLE = {TYPE}
-    ENC_NAME = "none"
-    IDHASH_NAME = "sha256"  # plain sha256(data), no key; blake3 is not supported for "none"
-
-    chunk_seed = 0
-    crypt_key = b""  # makes .derive_key() work, nothing secret here
-    id_key = b""  # makes .derive_key() work, nothing secret here
-
-    logically_encrypted = False
-
-    @classmethod
-    def create(cls, repository, args, **kw):
-        logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile" to enable encryption.')
-        return cls(repository)
-
-    @classmethod
-    def detect(cls, repository, manifest_data, *, other=False):
-        return cls(repository)
-
-    def id_hash(self, data):
-        return sha256(data).digest()
-
-    def encrypt(self, id, data, aad=b""):
-        return b"".join([self.TYPE_STR, data])
-
-    def decrypt(self, id, data, aad=b""):
-        self.assert_type(data[0], id)
-        return memoryview(data)[1:]
-
-
 class ID_HMAC_SHA_256:
     """
     Key mix-in class for using HMAC-SHA-256 for the id key.
@@ -477,6 +486,15 @@ class FlexiKey:
     FILE_ID = KEYFILE_ID
     STORAGE: ClassVar[str] = KeyBlobStorage.NO_STORAGE  # override in subclass
 
+    # this is a mix-in for KeyBase subclasses: it only uses these, KeyBase defines them.
+    # they are declared here (same types as there) so that a subclass in another module does not
+    # have to infer their types across the module import cycle.
+    chunk_seed: int
+    crypt_key: bytes
+    id_key: bytes
+    storage: str
+    target: Any
+
     # multiple-borg-keys state (a repository may have multiple borg keys, one per passphrase):
     _encrypted_key_label = None  # label read from the EncryptedKey envelope on decrypt
     _loaded_key_id = None  # key id (content digest / store name) of the borg key we unlocked
@@ -640,7 +658,9 @@ def create(cls, repository, args, *, other_key=None):
             # choose initial storage (keyfile or repokey) from --key-location (default: repokey).
             key.storage = KEY_LOCATIONS.get(getattr(args, "key_location", None), cls.STORAGE)
         if other_key is not None:
-            if isinstance(other_key, PlaintextKey):
+            if not other_key.has_secret_key:
+                # the "none-*" modes (and borg 1.x "none") have no key material to copy - and they
+                # need none: their chunk ids are unkeyed, so they all dedup identically anyway.
                 raise Error("Copying key material from an unencrypted repository is not possible.")
             if isinstance(key, AESKeyBase):
                 # user must use an AEADKeyBase subclass (AEAD modes with session keys)
@@ -988,10 +1008,176 @@ def remove_key(self, *, label=None, key_id=None, current=False):
         return victim
 
 
-class AuthenticatedKeyBase(AESKeyBase, FlexiKey):
+# ------------ not encrypted, but tagged: the "authenticated-*" and "none-*" modes ------------
+
+
+class MACKeyBase(KeyBase):
+    """
+    Base class of the modes that do not encrypt, but tag every repo object slot.
+
+    Envelope layout: TYPE(1) + reserved(1) + tag(32) + payload
+
+    The payload (the compressed chunk data resp. the packed metadata) is stored as-is, so anyone
+    can read it. The tag is computed over the envelope header, the AAD and the payload, so a read
+    detects any modification of them. RepoObj puts the object header (magic, format version, chunk
+    id), the meta/data slot tag and the chunk id into the AAD, thus the tag also binds a payload to
+    the specific object and slot it was written for: swapping the meta and data slot, moving a slot
+    to another object or changing the object header all fail verification.
+
+    The tag is deterministic - there is no nonce, no session and no other state to keep (a MAC does
+    not need any). Two repositories with the same key material therefore store byte-identical
+    objects for identical input, which allows deduplicating them on the filesystem level.
+
+    Subclasses supply the tag algorithm (mac()) and, if they are keyed, the tag key (tag_key).
+    """
+
+    encrypts = False  # these modes only tag the data, they do not encrypt it
+    IDHASH_IN_ENC_NAME = True  # the hash is what protects the data here, see key_creator
+
+    # TYPE + reserved + tag. reserved is currently always zero; it is authenticated by the tag,
+    # so it can carry a format flag later without changing the envelope size.
+    TAG_SIZE = 32
+    HEADER_SIZE = 2
+    PAYLOAD_OVERHEAD = HEADER_SIZE + TAG_SIZE
+
+    def mac(self, prefix: bytes, payload) -> bytes:
+        """Compute the authentication tag over prefix + payload, returning TAG_SIZE bytes."""
+        raise NotImplementedError
+
+    @property
+    def tag_key(self) -> bytes:
+        """The key the tag is computed with (unkeyed modes do not have one)."""
+        raise NotImplementedError
+
+    def init_ciphers(self, manifest_data=None):
+        # there is no cipher and no session state here (the tag is deterministic), we only check
+        # the type byte of the manifest object.
+        if manifest_data is not None:
+            self.assert_type(manifest_data[0])
+
+    def _tag_prefix(self, header: bytes, aad: bytes, id: bytes) -> bytes:
+        # everything the tag covers except the payload. The AAD is length-prefixed, so the
+        # boundary between it and the payload is unambiguous (the AAD has a fixed length today,
+        # but the tag must not depend on that staying true).
+        aad_full = aad + id
+        assert len(aad_full) <= 0xFFFF, "aad too long to be length-prefixed"
+        return b"".join([header, len(aad_full).to_bytes(2, "big"), aad_full])
+
+    def encrypt(self, id, data, aad=b""):
+        header = self.TYPE_STR + b"\0"  # TYPE + reserved
+        tag = self.mac(self._tag_prefix(header, aad, id), data)
+        return b"".join([header, tag, data])
+
+    def decrypt(self, id, data, aad=b""):
+        if len(data) < self.PAYLOAD_OVERHEAD:
+            raise IntegrityError(f"Chunk {bin_to_hex(id)}: truncated envelope")
+        self.assert_type(data[0], id)
+        obj = memoryview(data)
+        payload = obj[self.PAYLOAD_OVERHEAD :]
+        if self.has_secret_key and AUTHENTICATED_NO_KEY:
+            # we do not have the key material, so we can not verify the tag, see the
+            # BORG_WORKAROUNDS docs. Unkeyed modes verify even then - they need no key.
+            return payload
+        header = bytes(obj[: self.HEADER_SIZE])
+        tag = bytes(obj[self.HEADER_SIZE : self.PAYLOAD_OVERHEAD])
+        computed_tag = self.mac(self._tag_prefix(header, aad, id), payload)
+        if not hmac.compare_digest(computed_tag, tag):
+            raise IntegrityError(f"Chunk {bin_to_hex(id)}: envelope tag verification failed")
+        return payload
+
+
+class ChecksumKeyBase(MACKeyBase):
+    """
+    Base class of the "none-*" modes: no encryption, no key, no authentication.
+
+    The tag is an **unkeyed** hash, i.e. a checksum: it detects accidental corruption of the
+    payload, the metadata or the object header (including a read that returned the wrong bytes),
+    but it is no protection against malicious tampering - whoever changes the object can just
+    recompute the checksum. Detecting that requires a secret, see the "authenticated-*" modes.
+
+    Because there is no key, the chunk id is an unkeyed hash of the plaintext, too. Thus, all
+    repositories of this mode dedup identically and store byte-identical objects for identical
+    input, no key sharing needed.
+    """
+
+    STORAGE = KeyBlobStorage.NO_STORAGE
+    LOCATION_CONFIGURABLE = False
+
+    chunk_seed = 0
+    crypt_key = b""  # makes .derive_key() work, nothing secret here
+    id_key = b""  # makes .derive_key() work, nothing secret here
+
+    logically_encrypted = False
+    has_secret_key = False
+
+    # The checksum can be recomputed by anybody, so it is not an authentication and thus can not
+    # take over the read path authentication the id hash does. Different from the keyed classes
+    # below, verifying the chunk id must therefore never be skipped, see RepoObj.parse.
+    id_check_is_authentication = True
+
+    @classmethod
+    def create(cls, repository, args, **kw):
+        logger.info(
+            "Encryption NOT enabled.\n"
+            'Use "--encryption=repokey-aes-ocb" (or another encrypted mode) to enable encryption.'
+        )
+        return cls(repository)
+
+    @classmethod
+    def detect(cls, repository, manifest_data, *, other=False):
+        return cls(repository)
+
+
+class ChecksumKey(ChecksumKeyBase):
+    TYPE = KeyType.SHA256NONE
+    TYPES_ACCEPTABLE = {TYPE}
+    ENC_NAME = "none-sha256"
+    IDHASH_NAME = "sha256"
+
+    def id_hash(self, data):
+        return sha256(data).digest()
+
+    def mac(self, prefix, payload):
+        h = sha256(prefix)
+        h.update(payload)
+        return h.digest()
+
+
+class Blake3ChecksumKey(ChecksumKeyBase):
+    TYPE = KeyType.BLAKE3NONE
+    TYPES_ACCEPTABLE = {TYPE}
+    ENC_NAME = "none-blake3"
+    IDHASH_NAME = "blake3"
+
+    def id_hash(self, data):
+        # see ID_BLAKE3_256.id_hash about max_threads
+        max_threads = blake3.AUTO if len(data) >= get_blake3_mt_threshold() else 1
+        return blake3(data, max_threads=max_threads).digest(length=32)
+
+    def mac(self, prefix, payload):
+        max_threads = blake3.AUTO if len(payload) >= get_blake3_mt_threshold() else 1
+        h = blake3(prefix, max_threads=max_threads)
+        h.update(payload)
+        return h.digest(length=32)
+
+
+class AuthenticatedKeyBase(MACKeyBase, FlexiKey):
+    """
+    Base class of the "authenticated-*" modes: no encryption, but real authentication.
+
+    These modes have key material (in a borg key, like the encrypted modes have), so their tag is
+    a MAC: only somebody who has the borg key can compute it. Thus, a read detects tampering with
+    the payload, the metadata and the object header, no matter whether it was accidental or
+    malicious. The chunk ids are keyed hashes for the same reason.
+
+    The MAC key is derived from crypt_key (not from id_key: chunk ids are public, and related
+    repositories share the id key - see FlexiKey.create - which must not give them the ability to
+    forge each other's objects; "repo-create --other-repo --copy-crypt-key" opts into sharing it).
+    """
+
     # default storage; an individual key's actual storage is tracked per-instance in self.storage.
     STORAGE = KeyBlobStorage.REPO
-    # an authenticated-mode key has real key material (id/auth key) and a key blob, just no data
+    # an authenticated-mode key has real key material (id/MAC key) and a key blob, just no data
     # encryption. The blob may live as a keyfile or inside the repository, like the encrypted modes
     # (configurable via --key-location, changeable later via "borg key change-location").
     LOCATION_CONFIGURABLE = True
@@ -999,14 +1185,33 @@ class AuthenticatedKeyBase(AESKeyBase, FlexiKey):
     # It's only authenticated, not encrypted.
     logically_encrypted = False
 
+    # every read is authenticated by the envelope tag (which covers the chunk id via the AAD),
+    # independently of assert_id() - the same reasoning as for the AEAD keys, see
+    # AEADKeyBase.assert_id about what verifying the chunk id adds on top of that.
+    id_check_is_authentication = False
+
+    # domain for deriving the envelope MAC key from crypt_key, see tag_key. It is per MAC
+    # algorithm, so the classes below do not use the same key for different algorithms.
+    MAC_KEY_DOMAIN: ClassVar[bytes] = None  # override in subclass
+
+    _tag_key: bytes | None = None  # cache for the tag_key property
+
+    @property
+    def tag_key(self):
+        # derived lazily (and cached): a key instance may also be built by copying the key
+        # material onto a fresh instance, see "borg key change-location".
+        if self._tag_key is None:
+            self._tag_key = self.derive_key(salt=b"", domain=self.MAC_KEY_DOMAIN, size=32)
+        return self._tag_key
+
     def _load(self, key_data, passphrase):
         if AUTHENTICATED_NO_KEY:
-            # fake _load if we have no key or passphrase
-            NOPE = bytes(32)  # 256 bit all-zero
-            self.repository_id = NOPE
-            self.enc_key = NOPE
-            self.enc_hmac_key = NOPE
-            self.id_key = NOPE
+            # fake _load if we have no key or passphrase. The key material is all-zero and thus
+            # worthless, but these modes do not encrypt, so reading still works - decrypt() skips
+            # the tag verification and RepoObj.parse skips the chunk id check.
+            self.repository_id = bytes(32)
+            self.crypt_key = bytes(64)
+            self.id_key = bytes(32)
             self.chunk_seed = 0
             return True
         return super()._load(key_data, passphrase)
@@ -1025,32 +1230,46 @@ def save(self, target, passphrase, algorithm, create=False, label=None, replace=
         super().save(target, passphrase, algorithm, create=create, label=label, replace=replace)
         self.logically_encrypted = False
 
-    def init_ciphers(self, manifest_data=None):
-        if manifest_data is not None:
-            self.assert_type(manifest_data[0])
+    def init_from_given_data(self, *, crypt_key, id_key, chunk_seed):
+        assert len(crypt_key) == 32 + 32
+        assert len(id_key) == 32
+        assert isinstance(chunk_seed, int)
+        self.crypt_key = crypt_key
+        self.id_key = id_key
+        self.chunk_seed = chunk_seed
+        self._tag_key = None  # invalidate: it is derived from crypt_key
 
-    def encrypt(self, id, data, aad=b""):
-        return b"".join([self.TYPE_STR, data])
+    def init_from_random_data(self):
+        data = os.urandom(100)
+        chunk_seed = bytes_to_int(data[96:100])
+        # Convert to signed int32
+        if chunk_seed & 0x80000000:
+            chunk_seed = chunk_seed - 0xFFFFFFFF - 1
+        self.init_from_given_data(crypt_key=data[0:64], id_key=data[64:96], chunk_seed=chunk_seed)
 
-    def decrypt(self, id, data, aad=b""):
-        self.assert_type(data[0], id)
-        return memoryview(data)[1:]
 
+class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):
+    TYPE = KeyType.SHA256AUTHENTICATED
+    TYPES_ACCEPTABLE = {TYPE}
+    ENC_NAME = "authenticated-sha256"  # IDHASH_NAME = "sha256" via ID_HMAC_SHA_256 mix-in
+    MAC_KEY_DOMAIN = b"borg-repoobj-mac-hmac-sha256"
+
+    def mac(self, prefix, payload):
+        h = hmac.new(self.tag_key, prefix, sha256)
+        h.update(payload)
+        return h.digest()
 
-# legacy imports placed after FlexiKey/AESKeyBase/KeyBase/AuthenticatedKeyBase so those names are already
+
+# legacy imports placed after FlexiKey/AESKeyBase/KeyBase so those names are already
 # in the partial module when legacy/crypto/key.py imports them back during circular load
 from ..legacy.crypto.key import AESCTRKey, Blake2AESCTRKey  # noqa: F401
 from ..legacy.crypto.key import Blake2AuthenticatedKey  # noqa: F401
+from ..legacy.crypto.key import AuthenticatedKey as LegacyAuthenticatedKey  # noqa: E402
+from ..legacy.crypto.key import PlaintextKey as LegacyPlaintextKey  # noqa: E402
 from ..legacy.crypto.key import LEGACY_KEY_TYPES  # noqa: E402
 from ..legacy.crypto.key import ID_BLAKE2b_256  # noqa: F401
 
 
-class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):
-    TYPE = KeyType.AUTHENTICATED
-    TYPES_ACCEPTABLE = {TYPE}
-    ENC_NAME = "authenticated"  # IDHASH_NAME = "sha256" via ID_HMAC_SHA_256 mix-in
-
-
 # ------------ new crypto ------------
 
 
@@ -1073,7 +1292,15 @@ def id_hash(self, data):
 class Blake3AuthenticatedKey(ID_BLAKE3_256, AuthenticatedKeyBase):
     TYPE = KeyType.BLAKE3AUTHENTICATED
     TYPES_ACCEPTABLE = {TYPE}
-    ENC_NAME = "authenticated"  # IDHASH_NAME = "blake3" via ID_BLAKE3_256 mix-in
+    ENC_NAME = "authenticated-blake3"  # IDHASH_NAME = "blake3" via ID_BLAKE3_256 mix-in
+    MAC_KEY_DOMAIN = b"borg-repoobj-mac-blake3"
+
+    def mac(self, prefix, payload):
+        # see ID_BLAKE3_256.id_hash about max_threads
+        max_threads = blake3.AUTO if len(payload) >= get_blake3_mt_threshold() else 1
+        h = blake3(prefix, key=self.tag_key, max_threads=max_threads)
+        h.update(payload)
+        return h.digest(length=32)
 
 
 class AEADKeyBase(KeyBase):
@@ -1277,14 +1504,15 @@ class Blake3CHPOKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey):
 
 
 AVAILABLE_KEY_TYPES = (
-    # these are available encryption modes for new repositories
-    # not encrypted modes
-    PlaintextKey,
-    AuthenticatedKey,
-    # new crypto
-    Blake3AuthenticatedKey,
+    # these are available encryption modes for new repositories, ordered by descending security
+    # encrypted modes (the id hash is selected via --id-hash)
     AESOCBKey,
     CHPOKey,
     Blake3AESOCBKey,
     Blake3CHPOKey,
+    # not encrypted modes (the id hash is part of the mode name, see key_creator)
+    AuthenticatedKey,
+    Blake3AuthenticatedKey,
+    ChecksumKey,
+    Blake3ChecksumKey,
 )
diff --git a/src/borg/legacy/crypto/key.py b/src/borg/legacy/crypto/key.py
index e001de5693..387933023b 100644
--- a/src/borg/legacy/crypto/key.py
+++ b/src/borg/legacy/crypto/key.py
@@ -1,10 +1,11 @@
 import hmac
 import os
-from hashlib import pbkdf2_hmac
+from hashlib import pbkdf2_hmac, sha256
 
 from ...constants import *  # NOQA
 from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, hmac_sha256, blake2b_256
-from ...crypto.key import ID_HMAC_SHA_256, AESKeyBase, FlexiKey, AuthenticatedKeyBase, UnsupportedKeyFormatError
+from ...crypto.key import ID_HMAC_SHA_256, KeyBase, AESKeyBase, FlexiKey, UnsupportedKeyFormatError
+from ...crypto.key import AUTHENTICATED_NO_KEY
 from ...helpers import get_limited_unpacker, msgpack
 from ...item import EncryptedKey
 from .low_level import AES
@@ -87,6 +88,102 @@ def init_from_random_data(self):
         self.id_key = random_blake2b_256_key()
 
 
+# borg 1.x unencrypted modes ("none" and "authenticated"). Their repo object envelope is just the
+# type byte followed by the payload: nothing is authenticated by it, the only integrity check a read
+# has is the chunk id hash over the plaintext. borg 2 replaced them with the tagged envelope modes
+# (see MACKeyBase in borg.crypto.key), so these classes are read-only: they exist to read borg 1.x
+# repos, e.g. for "borg transfer --from-borg1". borg 2 never creates them (they are not in
+# AVAILABLE_KEY_TYPES) and refuses to use them for borg 2 repos (see key_factory).
+
+
+class PlaintextKey(KeyBase):
+    TYPE = KeyType.PLAINTEXT
+    TYPES_ACCEPTABLE = {TYPE}
+    ENC_NAME = "none"  # borg 1.x name of this mode; read-only (borg 1.x)
+    IDHASH_NAME = "sha256"
+
+    chunk_seed = 0
+    crypt_key = b""  # makes .derive_key() work, nothing secret here
+    id_key = b""  # makes .derive_key() work, nothing secret here
+
+    logically_encrypted = False
+    encrypts = False
+    has_secret_key = False  # unkeyed sha256 chunk ids, no key material at all
+
+    @classmethod
+    def create(cls, repository, args, **kw):
+        raise NotImplementedError("borg 2 does not create borg 1.x repositories.")
+
+    @classmethod
+    def detect(cls, repository, manifest_data, *, other=False):
+        return cls(repository)
+
+    def id_hash(self, data):
+        return sha256(data).digest()
+
+    def encrypt(self, id, data, aad=b""):
+        return b"".join([self.TYPE_STR, data])
+
+    def decrypt(self, id, data, aad=b""):
+        self.assert_type(data[0], id)
+        return memoryview(data)[1:]
+
+
+class AuthenticatedKeyBase(AESKeyBase, FlexiKey):
+    # default storage; an individual key's actual storage is tracked per-instance in self.storage.
+    STORAGE = KeyBlobStorage.REPO
+    # an authenticated-mode key has real key material (id/auth key) and a key blob, just no data
+    # encryption. The blob may live as a keyfile or inside the repository, like the encrypted modes.
+    LOCATION_CONFIGURABLE = True
+
+    # It's only authenticated, not encrypted.
+    logically_encrypted = False
+    encrypts = False
+
+    def _load(self, key_data, passphrase):
+        if AUTHENTICATED_NO_KEY:
+            # fake _load if we have no key or passphrase
+            NOPE = bytes(32)  # 256 bit all-zero
+            self.repository_id = NOPE
+            self.enc_key = NOPE
+            self.enc_hmac_key = NOPE
+            self.id_key = NOPE
+            self.chunk_seed = 0
+            return True
+        return super()._load(key_data, passphrase)
+
+    def load(self, target, passphrase):
+        success = super().load(target, passphrase)
+        self.logically_encrypted = False
+        return success
+
+    def load_any(self, passphrase):
+        success = super().load_any(passphrase)
+        self.logically_encrypted = False
+        return success
+
+    def save(self, target, passphrase, algorithm, create=False, label=None, replace=True):
+        super().save(target, passphrase, algorithm, create=create, label=label, replace=replace)
+        self.logically_encrypted = False
+
+    def init_ciphers(self, manifest_data=None):
+        if manifest_data is not None:
+            self.assert_type(manifest_data[0])
+
+    def encrypt(self, id, data, aad=b""):
+        return b"".join([self.TYPE_STR, data])
+
+    def decrypt(self, id, data, aad=b""):
+        self.assert_type(data[0], id)
+        return memoryview(data)[1:]
+
+
+class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):  # type: ignore[misc]
+    TYPE = KeyType.AUTHENTICATED
+    TYPES_ACCEPTABLE = {TYPE}
+    ENC_NAME = "authenticated"  # IDHASH_NAME = "sha256" via ID_HMAC_SHA_256 mix-in; read-only (borg 1.x)
+
+
 class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase):  # type: ignore[misc]
     TYPE = KeyType.BLAKE2AUTHENTICATED
     TYPES_ACCEPTABLE = {TYPE}
@@ -117,4 +214,4 @@ class Blake2AESCTRKey(Pbkdf2FileMixin, ID_BLAKE2b_256, AESKeyBase, FlexiKey):  #
     CIPHERSUITE = AES256_CTR_BLAKE2b
 
 
-LEGACY_KEY_TYPES = (AESCTRKey, Blake2AESCTRKey, Blake2AuthenticatedKey)
+LEGACY_KEY_TYPES = (AESCTRKey, Blake2AESCTRKey, PlaintextKey, AuthenticatedKey, Blake2AuthenticatedKey)
diff --git a/src/borg/repoobj.py b/src/borg/repoobj.py
index 4d570cd3f4..61e5b541d4 100644
--- a/src/borg/repoobj.py
+++ b/src/borg/repoobj.py
@@ -7,7 +7,7 @@
 from .helpers.errors import Error, IntegrityError
 from .compress import Compressor, LZ4_COMPRESSOR
 
-# Workaround for lost passphrase or key in "authenticated" or "authenticated-blake2" mode
+# Workaround for lost passphrase or key in the "authenticated-*" modes
 AUTHENTICATED_NO_KEY = "authenticated_no_key" in workarounds
 
 # The places where parse() verifies chunkid == id_hash(content) only if the user asks for it.
@@ -23,8 +23,9 @@
 # not verifying elsewhere defensible, so it must not be switchable.
 ASSERT_ID_PLACES_MANDATORY = ("verify_data",)  # borg check --verify-data
 # Default value of the BORG_ASSERT_ID env var: verify everywhere except on the (hot) general read
-# path - there, the AEAD authentication already covers what a repository could do to us, see
-# AEADKeyBase.assert_id.
+# path - there, the envelope authentication of the keyed modes already covers what a repository
+# could do to us, see AEADKeyBase.assert_id. (For the modes whose id check *is* the read path
+# authentication, this setting does not apply, see below.)
 BORG_ASSERT_ID_DEFAULT = ("repair", "transfer", "rechunk")
 # Reads through a RepoObj are attributed to this place unless a command sets another one.
 ASSERT_ID_PLACE_DEFAULT = "read"
@@ -34,8 +35,9 @@ def get_assert_id_places():
     """Determine the configurable places that shall verify the chunk id, see the BORG_ASSERT_ID docs.
 
     Note: this only decides for keys that authenticate reads independently of the id hash (the AEAD
-    ciphersuites, see KeyBase.id_check_is_authentication) - for all other keys, the id check is the
-    read path authentication itself and thus always happens, no matter what is configured here.
+    ciphersuites and the "authenticated-*" modes, see KeyBase.id_check_is_authentication) - for all
+    other keys, the id check is the read path authentication itself and thus always happens, no
+    matter what is configured here.
     The same is true for ASSERT_ID_PLACES_MANDATORY.
     """
     value = os.environ.get("BORG_ASSERT_ID")
@@ -252,19 +254,23 @@ def parse(
             compressor_cls, compression_level = Compressor.detect(compr_hdr)
             compressor = compressor_cls(level=compression_level)
             meta, data = compressor.decompress(dict(meta_compressed), data_compressed[:psize])
-            # For keys where the id check is the read-path authentication ("authenticated" and "none" mode),
-            # it always has to happen - skipping it would remove all integrity checking from reads.
-            # For the AEAD keys, the ciphertext is already authenticated for this specific chunk id (the id is
-            # in the AAD), so the id check only adds detection of chunks whose plaintext does not match their id
-            # - which only an evil/broken borg client that had the repo key could have written. Whether that
-            # extra full-plaintext hash pass is worth it depends on the place we read at, see BORG_ASSERT_ID.
+            # For keys where the id check is the read-path authentication (the "none-*" modes, whose
+            # envelope checksum is unkeyed and thus no authentication), it always has to happen -
+            # skipping it would remove all integrity checking from reads.
+            # For the keys that authenticate the envelope (the AEAD and the "authenticated-*" modes), the
+            # payload is already authenticated for this specific chunk id (the id is in the AAD), so the id
+            # check only adds detection of chunks whose plaintext does not match their id - which only an
+            # evil/broken borg client that had the repo key could have written. Whether that extra
+            # full-plaintext hash pass is worth it depends on the place we read at, see BORG_ASSERT_ID.
             place = assert_id_place if assert_id_place is not None else self.assert_id_place
             assert_id = (
                 self.key.id_check_is_authentication
                 or place in ASSERT_ID_PLACES_MANDATORY
                 or place in self.assert_id_places
             )
-            if assert_id and not AUTHENTICATED_NO_KEY:
+            # the authenticated_no_key workaround fakes key material it could not unlock, so checks that
+            # need that key material can not work. The unkeyed modes need none, so they keep checking.
+            if assert_id and not (AUTHENTICATED_NO_KEY and self.key.has_secret_key):
                 self.key.assert_id(id, data)
         else:
             meta, data = None, None
diff --git a/src/borg/security.py b/src/borg/security.py
index 5111280fa8..d76c762927 100644
--- a/src/borg/security.py
+++ b/src/borg/security.py
@@ -150,7 +150,6 @@ def assert_location_matches(self):
                 fd.write(repository_location)
 
     def assert_no_manifest_replay(self, manifest, key):
-        from .crypto.key import PlaintextKey
 
         try:
             with self.manifest_ts_file.open() as fd:
@@ -165,7 +164,9 @@ def assert_no_manifest_replay(self, manifest, key):
         logger.debug("security: determined newest manifest timestamp as %s", timestamp)
         # If repository is older than the cache or security dir something fishy is going on
         if timestamp and timestamp > manifest.timestamp:
-            if isinstance(key, PlaintextKey):
+            if not key.has_secret_key:
+                # unkeyed modes ("none-*", and borg 1.x "none"): the repository id is not derived
+                # from any secret, so two repositories can legitimately have the same id.
                 raise RepositoryIDNotUnique()
             else:
                 raise RepositoryReplay()
diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py
index 8da624c9da..cb2aee520e 100644
--- a/src/borg/testsuite/archive_test.py
+++ b/src/borg/testsuite/archive_test.py
@@ -10,7 +10,7 @@
 from . import rejected_dotdot_paths
 from ..cache import ChunkListEntry
 from ..constants import ROBJ_FILE_STREAM, zeros
-from ..crypto.key import PlaintextKey
+from ..crypto.key import ChecksumKey
 from ..archive import Archive, CacheChunkBuffer, DownloadPipeline, RobustUnpacker, valid_msgpacked_dict
 from ..archive import ITEM_KEYS, Statistics
 from ..archive import BackupOSError, backup_io, backup_io_iter, get_item_uid_gid
@@ -181,7 +181,7 @@ def test_stats_progress_json(stats):
 )
 def test_timestamp_parsing(monkeypatch, isoformat, expected):
     repository = Mock()
-    key = PlaintextKey(repository)
+    key = ChecksumKey(repository)
     manifest = Manifest(key, repository)
     a = Archive(manifest, "test", create=True)
     a.metadata = ArchiveItem(time=isoformat)
@@ -205,7 +205,7 @@ def add_chunk(self, id, meta, data, stats=None, ro_type=None):
 def test_cache_chunk_buffer():
     data = [Item(path="p1"), Item(path="p2")]
     cache = MockCache()
-    key = PlaintextKey(None)
+    key = ChecksumKey(None)
     chunks = CacheChunkBuffer(cache, key, None)
     for d in data:
         chunks.add(d)
@@ -222,7 +222,7 @@ def test_partial_cache_chunk_buffer():
     big = "0123456789abcdefghijklmnopqrstuvwxyz" * 25000
     data = [Item(path="full", target=big), Item(path="partial", target=big)]
     cache = MockCache()
-    key = PlaintextKey(None)
+    key = ChecksumKey(None)
     chunks = CacheChunkBuffer(cache, key, None)
     for d in data:
         chunks.add(d)
@@ -257,7 +257,7 @@ def test_download_pipeline_parsed_cache():
     # a content data stream may reference the same chunk many times (e.g. the all-zero
     # chunks of a sparse file): repeated chunks shall be parsed (decrypted, authenticated,
     # decompressed) only once, see issue #1678.
-    key = PlaintextKey(None)
+    key = ChecksumKey(None)
     repo_objs = RepoObj(key)
     # note: repeated, but not all-zero data, so it is not served via the zeros shortcut
     chunks_data = [b"foobar" * 100, b"idletone" * 125, b"barbaz" * 100]
@@ -290,7 +290,7 @@ def counting_parse(id, cdata, **kw):
 def test_download_pipeline_missing_chunk(replacement_chunk):
     # a chunk missing in the repository is either replaced by all-zero data of the
     # correct size, or reported as None - and never blows up on the size check.
-    key = PlaintextKey(None)
+    key = ChecksumKey(None)
     repo_objs = RepoObj(key)
     data = b"foobar" * 100
     id = repo_objs.id_hash(data)
@@ -304,7 +304,7 @@ def test_download_pipeline_missing_chunk(replacement_chunk):
 def test_download_pipeline_zero_chunks_served_locally():
     # repeated all-zero chunks (e.g. from the holes of a sparse file) shall be served
     # directly from the zeros constant, without repository access, see issue #1678.
-    key = PlaintextKey(None)
+    key = ChecksumKey(None)
     repo_objs = RepoObj(key)
     data = b"foobar" * 100
     data_id = repo_objs.id_hash(data)
diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py
index f9b881cc38..e5d083fda6 100644
--- a/src/borg/testsuite/archiver/check_cmd_test.py
+++ b/src/borg/testsuite/archiver/check_cmd_test.py
@@ -716,7 +716,7 @@ def test_repair_finish_flushes_pack_writer(archivers, request):
         assert not repository._pack_writer._pieces  # finish() stored it
 
 
-@pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none"]])
+@pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none-sha256"]])
 def test_verify_data(archivers, request, init_args):
     archiver = request.getfixturevalue(archivers)
     if archiver.get_kind() != "local":
@@ -809,7 +809,7 @@ def test_repair_wrong_item_metadata_chunk_content(archivers, request, monkeypatc
     assert "id verification failed" in output
 
 
-@pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none"]])
+@pytest.mark.parametrize("init_args", [["--encryption=aes256-ocb"], ["--encryption", "none-sha256"]])
 def test_corrupted_file_chunk(archivers, request, init_args):
     ## like test_verify_data, but also checks a repository-only check passes after repair and a plain
     ## archives check reports the missing chunk.
diff --git a/src/borg/testsuite/archiver/checks_test.py b/src/borg/testsuite/archiver/checks_test.py
index 9327feb9d9..4841265d91 100644
--- a/src/borg/testsuite/archiver/checks_test.py
+++ b/src/borg/testsuite/archiver/checks_test.py
@@ -46,7 +46,7 @@ def test_repository_swap_detection(archivers, request):
     repository_id = _extract_repository_id(archiver.repository_path)
     cmd(archiver, "create", "test", "input")
     shutil.rmtree(archiver.repository_path)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     _set_repository_id(archiver.repository_path, repository_id)
     assert repository_id == _extract_repository_id(archiver.repository_path)
     if archiver.FORK_DEFAULT:
@@ -61,7 +61,7 @@ def test_repository_swap_detection2(archivers, request):
     create_test_files(archiver.input_path)
     original_location = archiver.repository_location
     archiver.repository_location = original_location + "_unencrypted"
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     os.environ["BORG_PASSPHRASE"] = "passphrase"
     archiver.repository_location = original_location + "_encrypted"
     cmd(archiver, "repo-create", RK_ENCRYPTION)
@@ -83,7 +83,7 @@ def test_repository_swap_detection_no_cache(archivers, request):
     repository_id = _extract_repository_id(archiver.repository_path)
     cmd(archiver, "create", "test", "input")
     shutil.rmtree(archiver.repository_path)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     _set_repository_id(archiver.repository_path, repository_id)
     assert repository_id == _extract_repository_id(archiver.repository_path)
     cmd(archiver, "repo-delete", "--cache-only")
@@ -99,7 +99,7 @@ def test_repository_swap_detection2_no_cache(archivers, request):
     original_location = archiver.repository_location
     create_test_files(archiver.input_path)
     archiver.repository_location = original_location + "_unencrypted"
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     os.environ["BORG_PASSPHRASE"] = "passphrase"
     archiver.repository_location = original_location + "_encrypted"
     cmd(archiver, "repo-create", RK_ENCRYPTION)
@@ -175,7 +175,7 @@ def test_repository_move(archivers, request, monkeypatch):
 
 def test_unknown_unencrypted(archivers, request, monkeypatch):
     archiver = request.getfixturevalue(archivers)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     # Ok: repository is known
     cmd(archiver, "repo-info")
 
diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py
index 69995509d0..15c2f064e8 100644
--- a/src/borg/testsuite/archiver/create_cmd_test.py
+++ b/src/borg/testsuite/archiver/create_cmd_test.py
@@ -136,7 +136,7 @@ def test_archived_paths(archivers, request):
     # no leading slash in borg archives:
     archived_path = posix_path.lstrip("/")
     create_regular_file(archiver.input_path, "test")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "test", "input", posix_path)
     # "input" directory is recursed into, "input/test" is discovered and joined by borg's recursion.
     # posix_path was directly given as a cli argument and should end up as archive_path in the borg archive.
@@ -202,7 +202,7 @@ def test_create_duplicate_root(archivers, request):
     hl_b = os.path.join(path_b, "hardlink")
     create_regular_file(archiver.input_path, hl_a, contents=b"123456")
     os.link(hl_a, hl_b)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "test", "input", "input")  # give input twice!
     # test if created archive has 'input' contents twice:
     archive_list = cmd(archiver, "list", "test", "--json-lines")
@@ -218,7 +218,7 @@ def test_create_unreadable_parent(archiver):
     os.mkdir(root_dir)
     os.chmod(parent_dir, 0o111)  # --x--x--x == parent dir traversable, but not readable
     try:
-        cmd(archiver, "repo-create", "--encryption=none")
+        cmd(archiver, "repo-create", "--encryption=none-sha256")
         # issue #7746: we *can* read root_dir and we *can* traverse parent_dir, so this should work:
         cmd(archiver, "create", "test", root_dir)
     finally:
diff --git a/src/borg/testsuite/archiver/disk_full_test.py b/src/borg/testsuite/archiver/disk_full_test.py
index 8bf8a38ca4..130153e84f 100644
--- a/src/borg/testsuite/archiver/disk_full_test.py
+++ b/src/borg/testsuite/archiver/disk_full_test.py
@@ -51,7 +51,7 @@ def test_disk_full(test_pass, cmd_fixture, monkeypatch):
     input = os.path.join(DF_MOUNT, "input")
     shutil.rmtree(repo, ignore_errors=True)
     shutil.rmtree(input, ignore_errors=True)
-    rc, out = cmd_fixture(f"--repo={repo}", "repo-create", "--encryption=none")
+    rc, out = cmd_fixture(f"--repo={repo}", "repo-create", "--encryption=none-sha256")
     if rc != EXIT_SUCCESS:
         print("repo-create", rc, out)
     assert rc == EXIT_SUCCESS
diff --git a/src/borg/testsuite/archiver/extract_cmd_test.py b/src/borg/testsuite/archiver/extract_cmd_test.py
index 05ef2bff91..d2a3e302c9 100644
--- a/src/borg/testsuite/archiver/extract_cmd_test.py
+++ b/src/borg/testsuite/archiver/extract_cmd_test.py
@@ -462,7 +462,7 @@ def test_extract_hardlinks_twice(archivers, request):
     hl_b = os.path.join(path_b, "hardlink")
     create_regular_file(archiver.input_path, hl_a, contents=b"123456")
     os.link(hl_a, hl_b)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "test", "input", "input")  # give input twice!
     # now test extraction
     with changedir("output"):
@@ -703,7 +703,7 @@ def patched_setxattr_EACCES(*args, **kwargs):
 
     create_regular_file(archiver.input_path, "file")
     xattr.setxattr(b"input/file", b"user.attribute", b"value")
-    cmd(archiver, "repo-create", "-e", "none")
+    cmd(archiver, "repo-create", "-e", "none-sha256")
     cmd(archiver, "create", "test", "input")
     with changedir("output"):
         input_abspath = os.path.abspath("input/file")
@@ -731,7 +731,7 @@ def patched_setxattr_EACCES(*args, **kwargs):
 def test_extract_xattrs_resourcefork(archivers, request):
     archiver = request.getfixturevalue(archivers)
     create_regular_file(archiver.input_path, "file")
-    cmd(archiver, "repo-create", "-e", "none")
+    cmd(archiver, "repo-create", "-e", "none-sha256")
     input_path = os.path.abspath("input/file")
     xa_key, xa_value = b"com.apple.ResourceFork", b"whatshouldbehere"  # issue #7234
     xattr.setxattr(input_path.encode(), xa_key, xa_value)
@@ -770,7 +770,7 @@ def test_extract_restores_append_flag(archivers, request):
     if (platform.get_flags(src_path, st) & stat.UF_APPEND) == 0:
         pytest.skip("UF_APPEND not settable on this filesystem")
     # archive and extract
-    cmd(archiver, "repo-create", "-e", "none")
+    cmd(archiver, "repo-create", "-e", "none-sha256")
     cmd(archiver, "create", "test", "input")
     with changedir("output"):
         cmd(archiver, "extract", "test")
@@ -821,7 +821,7 @@ def patched_setxattr_EACCES(*args, **kwargs):
 
     create_regular_file(archiver.input_path, "file")
     xattr.setxattr(b"input/file", b"user.attribute%p", b"value")
-    cmd(archiver, "repo-create", "-e", "none")
+    cmd(archiver, "repo-create", "-e", "none-sha256")
     cmd(archiver, "create", "test", "input")
     with changedir("output"):
         with patch.object(xattr, "setxattr", patched_setxattr_EACCES):
@@ -841,7 +841,7 @@ def patched_setxattr_EACCES(*args, **kwargs):
 
     os.makedirs(os.path.join(archiver.input_path, "dir%p"))
     xattr.setxattr(b"input/dir%p", b"user.attribute", b"value")
-    cmd(archiver, "repo-create", "-e", "none")
+    cmd(archiver, "repo-create", "-e", "none-sha256")
     cmd(archiver, "create", "test", "input")
     with changedir("output"):
         with patch.object(xattr, "setxattr", patched_setxattr_EACCES):
diff --git a/src/borg/testsuite/archiver/key_cmds_test.py b/src/borg/testsuite/archiver/key_cmds_test.py
index 44beb8551e..c553761cc9 100644
--- a/src/borg/testsuite/archiver/key_cmds_test.py
+++ b/src/borg/testsuite/archiver/key_cmds_test.py
@@ -7,7 +7,7 @@
 from ...constants import *  # NOQA
 from ...constants import KeyBlobStorage
 from ...crypto.key import AESOCBKey, CHPOKey, Passphrase, is_keyfile, keyfile_parse
-from ...crypto.keymanager import RepoIdMismatch, NotABorgKeyFile
+from ...crypto.keymanager import RepoIdMismatch, NotABorgKeyFile, UnencryptedRepo
 from ...helpers import CommandError
 from ...helpers import bin_to_hex, hex_to_bin
 from ...helpers import msgpack
@@ -79,26 +79,26 @@ def test_change_location_to_b3repokey(archivers, request):
 def test_change_location_authenticated_to_keyfile(archivers, request):
     # authenticated mode does not encrypt, but it still has a key whose location is configurable.
     archiver = request.getfixturevalue(archivers)
-    cmd(archiver, "repo-create", "--encryption=authenticated")
+    cmd(archiver, "repo-create", "--encryption=authenticated-sha256")
     log = cmd(archiver, "repo-info")
-    assert "(repokey, authenticated, sha256)" in log
+    assert "(repokey, authenticated-sha256)" in log
     cmd(archiver, "key", "change-location", "keyfile")
     [key_filename] = os.listdir(archiver.keys_path)
     assert key_filename  # key blob now lives as a keyfile
     log = cmd(archiver, "repo-info")
-    assert "(keyfile, authenticated, sha256)" in log
+    assert "(keyfile, authenticated-sha256)" in log
 
 
 def test_change_location_authenticated_to_repokey(archivers, request):
     archiver = request.getfixturevalue(archivers)
-    cmd(archiver, "repo-create", "--encryption=authenticated", KF_LOCATION)
+    cmd(archiver, "repo-create", "--encryption=authenticated-sha256", KF_LOCATION)
     assert os.listdir(archiver.keys_path)  # key blob created as a keyfile
     log = cmd(archiver, "repo-info")
-    assert "(keyfile, authenticated, sha256)" in log
+    assert "(keyfile, authenticated-sha256)" in log
     cmd(archiver, "key", "change-location", "repokey")
     assert os.listdir(archiver.keys_path) == []  # keyfile removed after moving into the repo
     log = cmd(archiver, "repo-info")
-    assert "(repokey, authenticated, sha256)" in log
+    assert "(repokey, authenticated-sha256)" in log
 
 
 def test_keyfile_name_is_content_sha256(archivers, request):
@@ -138,6 +138,24 @@ def test_borg_key_file_env_keeps_explicit_path(archivers, request, monkeypatch):
     assert os.listdir(archiver.keys_path) == []
 
 
+@pytest.mark.parametrize("mode", ["none-sha256", "none-blake3"])
+def test_key_management_unavailable_for_keyless_repo(archivers, request, mode):
+    # the "none-*" modes have no key at all, so there is nothing to export/import.
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", f"--encryption={mode}")
+    export_file = archiver.output_path + "/exported"
+    if archiver.FORK_DEFAULT:
+        # a forked/binary run returns the exit code, not the exception (BORG_EXIT_CODES defaults
+        # to "modern", so that is the specific mcode of UnencryptedRepo, not the generic EXIT_ERROR).
+        cmd(archiver, "key", "export", export_file, exit_code=UnencryptedRepo.exit_mcode)
+        cmd(archiver, "key", "import", export_file, exit_code=UnencryptedRepo.exit_mcode)
+    else:
+        with pytest.raises(UnencryptedRepo):
+            cmd(archiver, "key", "export", export_file)
+        with pytest.raises(UnencryptedRepo):
+            cmd(archiver, "key", "import", export_file)
+
+
 def test_key_export_keyfile(archivers, request):
     archiver = request.getfixturevalue(archivers)
     export_file = archiver.output_path + "/exported"
diff --git a/src/borg/testsuite/archiver/lock_cmds_test.py b/src/borg/testsuite/archiver/lock_cmds_test.py
index 81e9dcccd8..68a3e1f03d 100644
--- a/src/borg/testsuite/archiver/lock_cmds_test.py
+++ b/src/borg/testsuite/archiver/lock_cmds_test.py
@@ -28,7 +28,7 @@ def test_with_lock(tmp_path):
     print("sys.path: %r" % sys.path)
     print("PYTHONPATH: %s" % env.get("PYTHONPATH", ""))
     print("PATH: %s" % env.get("PATH", ""))
-    command0 = "python3", "-m", "borg", "repo-create", "--encryption=none"
+    command0 = "python3", "-m", "borg", "repo-create", "--encryption=none-sha256"
     # Timings must be adjusted so that command1 keeps running while command2 tries to get the lock,
     # so that lock acquisition for command2 fails as the test expects it.
     lock_wait = 2
diff --git a/src/borg/testsuite/archiver/mount_cmds_test.py b/src/borg/testsuite/archiver/mount_cmds_test.py
index f86e50e76b..57e18fb91e 100644
--- a/src/borg/testsuite/archiver/mount_cmds_test.py
+++ b/src/borg/testsuite/archiver/mount_cmds_test.py
@@ -392,7 +392,7 @@ def wrapper(self, old_id, new_id):
     # Decorate
     Lock.migrate_lock = write_assert_data(Lock.migrate_lock)
     try:
-        cmd(archiver, "repo-create", "--encryption=none")
+        cmd(archiver, "repo-create", "--encryption=none-sha256")
         create_src_archive(archiver, "arch")
         mountpoint = os.path.join(archiver.tmpdir, "mountpoint")
         # In order that the decoration is kept for the borg mount process, we must not spawn, but actually fork;
diff --git a/src/borg/testsuite/archiver/recreate_cmd_test.py b/src/borg/testsuite/archiver/recreate_cmd_test.py
index 14b8e5dada..2f861a8b66 100644
--- a/src/borg/testsuite/archiver/recreate_cmd_test.py
+++ b/src/borg/testsuite/archiver/recreate_cmd_test.py
@@ -66,7 +66,7 @@ def test_recreate_exclude_keep_tagged(archivers, request):
 @pytest.mark.skipif(not are_hardlinks_supported(), reason="hard links not supported")
 def test_recreate_hardlinked_tags(archivers, request):  # test for issue #4911
     archiver = request.getfixturevalue(archivers)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     create_regular_file(
         archiver.input_path, "file1", contents=CACHE_TAG_CONTENTS
     )  # "wrong" filename, but correct tag contents
diff --git a/src/borg/testsuite/archiver/repo_create_cmd_test.py b/src/borg/testsuite/archiver/repo_create_cmd_test.py
index 3c8fb56fb4..4cbb0171a4 100644
--- a/src/borg/testsuite/archiver/repo_create_cmd_test.py
+++ b/src/borg/testsuite/archiver/repo_create_cmd_test.py
@@ -6,7 +6,7 @@
 from ...helpers.errors import Error, CancelledByUser
 from ...constants import *  # NOQA
 from ...crypto.key import FlexiKey
-from . import cmd, generate_archiver_tests, RK_ENCRYPTION, KF_ENCRYPTION, KF_LOCATION
+from . import cmd, create_src_archive, generate_archiver_tests, RK_ENCRYPTION, KF_ENCRYPTION, KF_LOCATION
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary")  # NOQA
 
@@ -43,9 +43,13 @@ def test_repo_create_requires_encryption_option(archivers, request):
         (["--encryption=aes256-ocb", "--id-hash=blake3"], "Yes (repokey, aes256-ocb, blake3)"),
         (["--encryption=chacha20-poly1305"], "Yes (repokey, chacha20-poly1305, sha256)"),
         (["--encryption=chacha20-poly1305", "--id-hash=blake3"], "Yes (repokey, chacha20-poly1305, blake3)"),
-        (["--encryption=authenticated"], "No (repokey, authenticated, sha256)"),
-        (["--encryption=authenticated", "--id-hash=blake3"], "No (repokey, authenticated, blake3)"),
-        (["--encryption=none"], "No"),
+        # the modes that do not encrypt name their id hash themselves, --id-hash does not apply
+        (["--encryption=authenticated-sha256"], "No (repokey, authenticated-sha256)"),
+        (["--encryption=authenticated-blake3"], "No (repokey, authenticated-blake3)"),
+        (["--encryption=none-sha256"], "No (none-sha256)"),
+        (["--encryption=none-blake3"], "No (none-blake3)"),
+        # giving the matching --id-hash in addition is accepted
+        (["--encryption=none-blake3", "--id-hash=blake3"], "No (none-blake3)"),
     ],
 )
 def test_repo_create_encryption_id_hash_combinations(archivers, request, extra_args, expected):
@@ -55,10 +59,19 @@ def test_repo_create_encryption_id_hash_combinations(archivers, request, extra_a
     assert expected in info
 
 
-def test_repo_create_none_rejects_blake3(archivers, request):
-    # "none" (plaintext) has no key, so it only supports the sha256 id-hash.
+@pytest.mark.parametrize("mode", ["none", "authenticated"])
+def test_repo_create_rejects_bare_unencrypted_mode_names(archivers, request, mode):
+    # these modes always name their id hash now, e.g. "none-sha256", see #9104.
     archiver = request.getfixturevalue(archivers)
-    arg = ("repo-create", "--encryption=none", "--id-hash=blake3")
+    cmd(archiver, "repo-create", f"--encryption={mode}", exit_code=2)
+
+
+@pytest.mark.parametrize("mode", ["none-sha256", "authenticated-blake3"])
+def test_repo_create_rejects_conflicting_id_hash(archivers, request, mode):
+    # the id hash of these modes is part of the mode name, so a contradicting --id-hash is an error.
+    archiver = request.getfixturevalue(archivers)
+    conflicting = "blake3" if mode.endswith("sha256") else "sha256"
+    arg = ("repo-create", f"--encryption={mode}", f"--id-hash={conflicting}")
     if archiver.FORK_DEFAULT:
         cmd(archiver, *arg, exit_code=2)
     else:
@@ -72,6 +85,35 @@ def test_repo_create_rejects_legacy_combined_mode(archivers, request):
     cmd(archiver, "repo-create", "--encryption=blake3-aes-ocb", exit_code=2)
 
 
+def test_repo_create_related_authenticated_repos_store_identical_objects(archivers, request, monkeypatch):
+    # The envelope MAC key of the "authenticated-*" modes is derived from crypt_key, and the tag is
+    # deterministic, so two related repositories created with --copy-crypt-key store byte-identical
+    # objects for identical input (which allows deduplicating them on the filesystem level).
+    # Without --copy-crypt-key, crypt_key is a fresh random key and the objects differ.
+    archiver = request.getfixturevalue(archivers)
+    src_location = archiver.repository_location
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_src_archive(archiver, "arch")
+    monkeypatch.setenv("BORG_OTHER_PASSPHRASE", os.environ["BORG_PASSPHRASE"])
+
+    def transferred_objects(suffix, *extra_args):
+        archiver.repository_location = archiver.repository_path = src_location + suffix
+        cmd(archiver, "repo-create", "--encryption=authenticated-sha256", f"--other-repo={src_location}", *extra_args)
+        cmd(archiver, "transfer", f"--other-repo={src_location}")
+        objects = []
+        for dirpath, _, filenames in os.walk(os.path.join(archiver.repository_path, "packs")):
+            for filename in sorted(filenames):
+                with open(os.path.join(dirpath, filename), "rb") as fd:
+                    objects.append(fd.read())
+        assert objects  # we did transfer something
+        return sorted(objects)
+
+    same_key = transferred_objects("dst1", "--copy-crypt-key")
+    same_key_again = transferred_objects("dst2", "--copy-crypt-key")
+    assert same_key == same_key_again
+    assert transferred_objects("dst3") != same_key  # a fresh crypt_key gives different tags
+
+
 def test_repo_create_refuse_to_overwrite_keyfile(archivers, request, monkeypatch):
     #  BORG_KEY_FILE=something borg repo-create should quit if "something" already exists.
     #  See: https://github.com/borgbackup/borg/pull/6046
diff --git a/src/borg/testsuite/archiver/return_codes_test.py b/src/borg/testsuite/archiver/return_codes_test.py
index 0d4fd545ab..6e22ff4937 100644
--- a/src/borg/testsuite/archiver/return_codes_test.py
+++ b/src/borg/testsuite/archiver/return_codes_test.py
@@ -10,7 +10,7 @@
 
 def test_return_codes(archivers, request):
     archiver = request.getfixturevalue(archivers)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "archive", "input")
     with changedir("output"):
         cmd(archiver, "extract", "archive")
diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py
index a40c224866..3121e9a549 100644
--- a/src/borg/testsuite/archiver/tar_cmds_test.py
+++ b/src/borg/testsuite/archiver/tar_cmds_test.py
@@ -110,7 +110,7 @@ def test_import_tar(archivers, request, tar_format="PAX"):
     archiver = request.getfixturevalue(archivers)
     create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
     os.unlink("input/flagfile")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "src", "input")
     cmd(archiver, "export-tar", "src", "simple.tar", f"--tar-format={tar_format}")
     cmd(archiver, "import-tar", "dst", "simple.tar")
@@ -129,7 +129,7 @@ def test_import_unusual_tar(archivers, request):
     # ./foo//bar
     # ./
     tar_archive = os.path.join(os.path.dirname(__file__), "unusual_paths.tar")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "import-tar", "dst", tar_archive)
     files = cmd(archiver, "list", "dst", "--format", "{path}{NL}").splitlines()
     assert set(files) == {"foobar", "bar", "foo2", "foo/bar", "."}
@@ -143,7 +143,7 @@ def test_import_tar_with_dotdot(archivers, request):
     # Contains this file:
     # ../../../../etc/shadow
     tar_archive = os.path.join(os.path.dirname(__file__), "dotdot_path.tar")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     with pytest.raises(ValueError, match="unexpected '..' element in path '../../../../etc/shadow'"):
         cmd(archiver, "import-tar", "dst", tar_archive, exit_code=2)
 
@@ -153,7 +153,7 @@ def test_import_tar_gz(archivers, request, tar_format="GNU"):
     archiver = request.getfixturevalue(archivers)
     create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
     os.unlink("input/flagfile")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "src", "input")
     cmd(archiver, "export-tar", "src", "simple.tgz", f"--tar-format={tar_format}")
     cmd(archiver, "import-tar", "dst", "simple.tgz")
@@ -168,7 +168,7 @@ def test_export_import_tar_zst(archivers, request, suffix):
     archiver = request.getfixturevalue(archivers)
     create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
     os.unlink("input/flagfile")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "src", "input")
     cmd(archiver, "export-tar", "src", f"simple.{suffix}")
     with open(f"simple.{suffix}", "rb") as fd:
@@ -186,7 +186,7 @@ def test_export_import_tar_zst_mt(archivers, request, monkeypatch):
     monkeypatch.setattr("borg.compress._zstd_mt_workers", None)  # drop the cache
     create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
     os.unlink("input/flagfile")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "src", "input")
     cmd(archiver, "export-tar", "src", "simple.tar.zst")
     cmd(archiver, "import-tar", "dst", "simple.tar.zst")
@@ -217,7 +217,7 @@ def test_tar_filter_zstd_external(archivers, request):
     archiver = request.getfixturevalue(archivers)
     create_test_files(archiver.input_path, create_hardlinks=False)  # hard links become separate files
     os.unlink("input/flagfile")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "src", "input")
     cmd(archiver, "export-tar", "src", "simple.tar.zst", "--tar-filter=zstd")
     with open("simple.tar.zst", "rb") as fd:
@@ -247,7 +247,7 @@ def test_import_concatenated_tar_with_ignore_zeros(archivers, request):
             # Clean up for assert_dirs_equal.
             os.unlink("the_rest.tar")
 
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "import-tar", "--ignore-zeros", "dst", "input/concatenated.tar")
     # Clean up for assert_dirs_equal.
     os.unlink("input/concatenated.tar")
@@ -272,7 +272,7 @@ def test_import_concatenated_tar_without_ignore_zeros(archivers, request):
             with open("the_rest.tar", "rb") as the_rest:
                 concatenated.write(the_rest.read())
             os.unlink("the_rest.tar")
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "import-tar", "dst", "input/concatenated.tar")
 
     with changedir(archiver.output_path):
@@ -300,7 +300,7 @@ def test_import_tar_with_dotslash_paths(archivers, request):
         assert "./dir/file" in tar_content
 
     # Import the tar file into a Borg repository
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "import-tar", "dotslash", "input/dotslash.tar")
 
     # List the archive contents and verify no paths start with './'
@@ -314,7 +314,7 @@ def test_roundtrip_pax_borg(archivers, request):
     archiver = request.getfixturevalue(archivers)
     create_test_files(archiver.input_path)
     os.remove("input/flagfile")  # this would be automagically excluded due to NODUMP
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "src", "input")
     cmd(archiver, "export-tar", "src", "simple.tar", "--tar-format=BORG")
     cmd(archiver, "import-tar", "dst", "simple.tar")
@@ -331,7 +331,7 @@ def test_roundtrip_pax_xattrs(archivers, request):
     original_path = os.path.join(archiver.input_path, "file")
     xa_key, xa_value = b"user.xattrtest", b"not valid utf-8: \xff"
     xattr.setxattr(original_path.encode(), xa_key, xa_value)
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "src", "input")
     cmd(archiver, "export-tar", "src", "xattrs.tar", "--tar-format=PAX")
     cmd(archiver, "import-tar", "dst", "xattrs.tar")
@@ -393,7 +393,7 @@ def set_acl(path, access=None, default=None):
         pytest.skip("ACLs not supported or not working correctly")
 
     # 2. Create a Borg archive
-    cmd(archiver, "repo-create", "--encryption=none")
+    cmd(archiver, "repo-create", "--encryption=none-sha256")
     cmd(archiver, "create", "original", "input")
 
     # 3. export-tar this archive to a tar file
diff --git a/src/borg/testsuite/benchmark_test.py b/src/borg/testsuite/benchmark_test.py
index 9fd93e3505..fa798eb177 100644
--- a/src/borg/testsuite/benchmark_test.py
+++ b/src/borg/testsuite/benchmark_test.py
@@ -27,7 +27,7 @@ def repo_url(request, tmpdir, monkeypatch):
     tmpdir.remove(rec=1)
 
 
-@pytest.fixture(params=["none", "aes256-ocb"])
+@pytest.fixture(params=["none-sha256", "aes256-ocb"])
 def repo(request, cmd_fixture, repo_url):
     cmd_fixture(f"--repo={repo_url}", "repo-create", "--encryption", request.param)
     return repo_url
diff --git a/src/borg/testsuite/cockpit_test.py b/src/borg/testsuite/cockpit_test.py
index 8a13a2e9e0..d2eeba240e 100644
--- a/src/borg/testsuite/cockpit_test.py
+++ b/src/borg/testsuite/cockpit_test.py
@@ -24,7 +24,7 @@ def test_cockpit_app_create_archive(tmp_path):
     for i in range(5000):
         (input_path / f"test{i}.txt").write_text(f"content {i}")
 
-    subprocess.run(["borg", "-r", str(repo_path), "repo-create", "--encryption", "none"], check=True)
+    subprocess.run(["borg", "-r", str(repo_path), "repo-create", "--encryption", "none-sha256"], check=True)
 
     async def run():
         app = BorgCockpitApp()
diff --git a/src/borg/testsuite/crypto/crypto_test.py b/src/borg/testsuite/crypto/crypto_test.py
index 14fdfa62fa..c548336c02 100644
--- a/src/borg/testsuite/crypto/crypto_test.py
+++ b/src/borg/testsuite/crypto/crypto_test.py
@@ -9,7 +9,7 @@
 from ...crypto.low_level import hmac_sha256
 from ...legacy.crypto.low_level import AES
 from hashlib import sha256
-from ...crypto.key import CHPOKey, AESOCBKey, KeyBase, PlaintextKey
+from ...crypto.key import CHPOKey, AESOCBKey, KeyBase, ChecksumKey
 from ...legacy.crypto.key import AESCTRKey as LegacyAESCTRKey
 from ...helpers import msgpack, bin_to_hex
 
@@ -320,11 +320,11 @@ def __init__(self, crypt_key, id_key):
             self.id_key = id_key
 
     def test_derive_key_with_plaintext_key(self):
-        """Test derive_key with PlaintextKey (empty crypt_key)"""
-        key = PlaintextKey(None)
+        """Test derive_key with ChecksumKey (empty crypt_key)"""
+        key = ChecksumKey(None)
         salt, domain, size = b"salt", b"domain", 16
 
-        # PlaintextKey has an empty crypt_key, so the derived key should be based on salt and domain only
+        # ChecksumKey has an empty crypt_key, so the derived key should be based on salt and domain only
         derived_key = key.derive_key(salt=salt, domain=domain, size=size)
         expected = sha256(b"" + salt + domain).digest()[:size]
         self.assert_equal(derived_key, expected)
diff --git a/src/borg/testsuite/crypto/key_test.py b/src/borg/testsuite/crypto/key_test.py
index f6afd67347..9d370c79c3 100644
--- a/src/borg/testsuite/crypto/key_test.py
+++ b/src/borg/testsuite/crypto/key_test.py
@@ -6,21 +6,22 @@
 import pytest
 
 from ...crypto.key import BLAKE3_MT_THRESHOLD_KIB, get_blake3_mt_threshold
-from ...crypto.key import PlaintextKey, AuthenticatedKey, Blake2AuthenticatedKey, keyfile_parse
-from ...crypto.key import AESCTRKey, Blake2AESCTRKey
+from ...crypto.key import ChecksumKey, Blake3ChecksumKey, keyfile_parse
+from ...crypto.key import AuthenticatedKey, Blake3AuthenticatedKey
+from ...crypto.key import AESCTRKey, Blake2AESCTRKey, Blake2AuthenticatedKey
+from ...crypto.key import LegacyPlaintextKey, LegacyAuthenticatedKey
 from ...crypto.key import AEADKeyBase
 from ...crypto.key import AESOCBKey, CHPOKey, Blake3AESOCBKey, Blake3CHPOKey
 from ...crypto.key import AES_OCB_MAX_SESSION_BLOCKS
-from ...crypto.key import Blake3AuthenticatedKey
 from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, ID_BLAKE3_256
-from ...crypto.key import UnsupportedManifestError, UnsupportedKeyFormatError
+from ...crypto.key import UnsupportedManifestError, UnsupportedKeyFormatError, UnsupportedPayloadError
 from ...crypto.key import identify_key
 from ...crypto.low_level import IntegrityError as IntegrityErrorBase
 from ...helpers import Error
 from ...helpers import IntegrityError
 from ...helpers import Location
 from ...helpers import msgpack
-from ...constants import KEY_ALGORITHMS
+from ...constants import KEY_ALGORITHMS, KeyBlobStorage, KeyType, ROBJ_MANIFEST
 from ...helpers import hex_to_bin, bin_to_hex
 
 
@@ -87,11 +88,12 @@ def keys_dir(self, request, monkeypatch, tmpdir):
         params=(
             # keyfile and repokey are no longer separate classes (storage is a per-key property),
             # so each crypto suite appears once here.
-            # not encrypted
-            PlaintextKey,
+            # not encrypted, but tagged
+            ChecksumKey,
+            Blake3ChecksumKey,
             AuthenticatedKey,
             Blake3AuthenticatedKey,
-            # legacy crypto
+            # legacy crypto (read-only, borg 1.x)
             AESCTRKey,
             Blake2AESCTRKey,
             Blake2AuthenticatedKey,
@@ -128,13 +130,21 @@ def load_key(self):
             # is storage-agnostic now and always probes repo candidates, even for keyfile keys.
             return getattr(self, "key_data", b"")
 
-    def test_plaintext(self):
-        key = PlaintextKey.create(None, None)
+    def test_none_sha256(self):
+        key = ChecksumKey.create(None, None)
         chunk = b"foo"
         id = key.id_hash(chunk)
+        # the chunk id of the "none-*" modes is the plain (unkeyed) hash of the chunk
         assert bin_to_hex(id) == "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
         assert chunk == key.decrypt(id, key.encrypt(id, chunk))
 
+    def test_none_blake3(self):
+        key = Blake3ChecksumKey.create(None, None)
+        chunk = b"foo"
+        id = key.id_hash(chunk)
+        assert bin_to_hex(id) == "04e0bb39f30b1a3feb89f536c93be15055482df748674b00d26e5a75777702e9"
+        assert chunk == key.decrypt(id, key.encrypt(id, chunk))
+
     def test_keyfile(self, monkeypatch, keys_dir):
         monkeypatch.setenv("BORG_PASSPHRASE", "test")
         key = AESCTRKey.create(self.MockRepository(), self.kf_args())
@@ -292,10 +302,13 @@ def test_authenticated_encrypt(self, monkeypatch):
         plaintext = b"123456789"
         id = key.id_hash(plaintext)
         authenticated = key.encrypt(id, plaintext)
-        # 0x07 is the key TYPE.
-        assert authenticated == b"\x07" + plaintext
+        # TYPE(1) + reserved(1) + tag(32) + payload, see MACKeyBase
+        assert authenticated[0:2] == b"\x60\x00"
+        assert authenticated[34:] == plaintext
+        assert key.decrypt(id, authenticated) == plaintext
 
     def test_blake2_authenticated_encrypt(self, monkeypatch):
+        # borg 1.x mode, read-only: its envelope is just the type byte plus the payload.
         monkeypatch.setenv("BORG_PASSPHRASE", "test")
         key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
         assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash
@@ -314,8 +327,9 @@ def test_blake3_authenticated_encrypt(self, monkeypatch):
         plaintext = b"123456789"
         id = key.id_hash(plaintext)
         authenticated = key.encrypt(id, plaintext)
-        # 0x50 is the key TYPE.
-        assert authenticated == b"\x50" + plaintext
+        assert authenticated[0:2] == b"\x70\x00"
+        assert authenticated[34:] == plaintext
+        assert key.decrypt(id, authenticated) == plaintext
 
     def test_blake3_mt_threshold_from_env(self, monkeypatch):
         # the env var gives the threshold in KiB, get_blake3_mt_threshold() returns bytes
@@ -365,6 +379,190 @@ def test_round_trip_archive(self, key):
         assert "tam" not in unpacked  # legacy
 
 
+class TestMACEnvelope:
+    """The tagged envelope of the modes that do not encrypt, see MACKeyBase."""
+
+    # fixed key material, so the tests can check exact envelope bytes
+    CRYPT_KEY = bytes(range(64))
+    ID_KEY = bytes(range(100, 132))
+
+    ALL_CLASSES = (ChecksumKey, Blake3ChecksumKey, AuthenticatedKey, Blake3AuthenticatedKey)
+    KEYED_CLASSES = (AuthenticatedKey, Blake3AuthenticatedKey)
+    UNKEYED_CLASSES = (ChecksumKey, Blake3ChecksumKey)
+
+    class MockRepository:
+        id = bytes(32)
+        version = 2
+
+    def make_key(self, cls, crypt_key=None):
+        key = cls(self.MockRepository())
+        if cls.has_secret_key:
+            key.init_from_given_data(crypt_key=crypt_key or self.CRYPT_KEY, id_key=self.ID_KEY, chunk_seed=0)
+        return key
+
+    @pytest.fixture(params=ALL_CLASSES)
+    def key(self, request):
+        return self.make_key(request.param)
+
+    def test_envelope_layout(self, key):
+        payload = b"payload"
+        id = key.id_hash(payload)
+        envelope = key.encrypt(id, payload, aad=b"aad")
+        assert envelope[0] == key.TYPE
+        assert envelope[1] == 0  # reserved
+        assert len(envelope) == key.PAYLOAD_OVERHEAD + len(payload) == 34 + len(payload)
+        assert envelope[34:] == payload  # the payload is stored as-is, these modes do not encrypt
+        assert key.decrypt(id, envelope, aad=b"aad") == payload
+
+    def test_envelope_is_deterministic(self, key):
+        # no nonce, no session: same input -> same bytes (see MACKeyBase)
+        payload = b"payload"
+        id = key.id_hash(payload)
+        assert key.encrypt(id, payload, aad=b"aad") == key.encrypt(id, payload, aad=b"aad")
+
+    @pytest.mark.parametrize("cls", ALL_CLASSES)
+    def test_format_is_stable(self, cls):
+        # golden vectors: these bytes must not change silently, they are an on-disk format.
+        expected = {
+            ChecksumKey: "80001ab01692fad0f0b89f27983cbf51b1bd20ab73de736aea83363b749ee6ba314f",
+            Blake3ChecksumKey: "90008dec559eda6de95536f198f6b54b3b2728678c94ca1b3f0b77cf37e5b11c9b7c",
+            AuthenticatedKey: "600033ecaaf8c34d4142fa502278986c8f49efbcbf879de16d3c2767e1252bfb1994",
+            Blake3AuthenticatedKey: "70006de48e2139f8995790ec81b256e87f40ef5810d140d28396a8d07734ab05a358",
+        }[cls]
+        key = self.make_key(cls)
+        plaintext = b"123456789"
+        envelope = key.encrypt(key.id_hash(plaintext), plaintext, aad=b"")
+        assert bin_to_hex(envelope) == expected + bin_to_hex(plaintext)
+
+    @pytest.mark.parametrize("cls", UNKEYED_CLASSES)
+    def test_unkeyed_modes_are_repo_independent(self, cls):
+        # no key material at all, so any two repositories of such a mode store identical objects
+        # for identical input - that is what allows deduplicating them on the filesystem level.
+        key1, key2 = self.make_key(cls), self.make_key(cls)
+        payload = b"payload"
+        id = key1.id_hash(payload)
+        assert key1.id_hash(payload) == key2.id_hash(payload)
+        assert key1.encrypt(id, payload, aad=b"aad") == key2.encrypt(id, payload, aad=b"aad")
+
+    @pytest.mark.parametrize("cls", KEYED_CLASSES)
+    def test_keyed_modes_depend_on_crypt_key(self, cls):
+        # the tag key is derived from crypt_key: same crypt_key -> same objects (that is what
+        # "repo-create --other-repo --copy-crypt-key" gives), different crypt_key -> different tag.
+        payload = b"payload"
+        same = [self.make_key(cls), self.make_key(cls)]
+        other = self.make_key(cls, crypt_key=bytes(range(1, 65)))
+        id = same[0].id_hash(payload)
+        assert same[0].encrypt(id, payload, aad=b"aad") == same[1].encrypt(id, payload, aad=b"aad")
+        assert same[0].encrypt(id, payload, aad=b"aad") != other.encrypt(id, payload, aad=b"aad")
+        # a key that can not verify the tag must not be able to read
+        with pytest.raises(IntegrityError):
+            other.decrypt(id, same[0].encrypt(id, payload, aad=b"aad"), aad=b"aad")
+
+    def test_tampered_envelope_detected(self, key):
+        # every single byte of the envelope is covered by the tag
+        payload = b"0123456789"
+        id = key.id_hash(payload)
+        envelope = key.encrypt(id, payload, aad=b"aad")
+        for offset in range(len(envelope)):
+            tampered = bytearray(envelope)
+            tampered[offset] ^= 64  # stays within TYPES_ACCEPTABLE if it hits the type byte
+            with pytest.raises(IntegrityError):
+                key.decrypt(id, bytes(tampered), aad=b"aad")
+
+    def test_tampered_aad_detected(self, key):
+        # the AAD carries the object header, the meta/data slot tag and the chunk id, see RepoObj
+        payload = b"payload"
+        id = key.id_hash(payload)
+        envelope = key.encrypt(id, payload, aad=b"aadM")
+        with pytest.raises(IntegrityError):
+            key.decrypt(id, envelope, aad=b"aadD")  # e.g. meta and data slot swapped
+        other_id = key.id_hash(b"other payload")
+        with pytest.raises(IntegrityError):
+            key.decrypt(other_id, envelope, aad=b"aadM")  # chunk taken from another object
+
+    def test_truncated_envelope_detected(self, key):
+        payload = b"payload"
+        id = key.id_hash(payload)
+        envelope = key.encrypt(id, payload, aad=b"aad")
+        for length in range(key.PAYLOAD_OVERHEAD):  # cut into the header/tag
+            with pytest.raises(IntegrityError):
+                key.decrypt(id, envelope[:length], aad=b"aad")
+        with pytest.raises(IntegrityError):  # cut into the payload
+            key.decrypt(id, envelope[:-1], aad=b"aad")
+
+    @pytest.mark.parametrize("cls", ALL_CLASSES)
+    def test_flags(self, cls):
+        key = self.make_key(cls)
+        assert not key.encrypts
+        assert not key.logically_encrypted
+        assert key.IDHASH_IN_ENC_NAME
+        assert key.ENC_NAME.endswith("-" + key.IDHASH_NAME)
+        if cls in self.KEYED_CLASSES:
+            assert key.has_secret_key
+            # the envelope tag authenticates every read, so verifying the chunk id on top of that
+            # is optional (see BORG_ASSERT_ID), like for the AEAD modes.
+            assert not key.id_check_is_authentication
+            assert key.STORAGE == KeyBlobStorage.REPO
+            assert key.LOCATION_CONFIGURABLE
+        else:
+            assert not key.has_secret_key
+            # the checksum can be recomputed by anybody, thus it is no authentication: the chunk id
+            # check is the only one left and must never be skipped.
+            assert key.id_check_is_authentication
+            assert key.STORAGE == KeyBlobStorage.NO_STORAGE
+            assert not key.LOCATION_CONFIGURABLE
+
+    @pytest.mark.parametrize("cls", ALL_CLASSES)
+    def test_type_bytes_are_distinct(self, cls):
+        # identify_key must be able to tell the modes apart by the first envelope byte
+        key = self.make_key(cls)
+        envelope = key.encrypt(key.id_hash(b"x"), b"x")
+        assert identify_key(envelope) is cls
+
+    def test_authenticated_no_key_workaround(self, monkeypatch):
+        # without the key material, the keyed modes can not verify the tag - but they can still
+        # read the data (that is the point of the workaround, see BORG_WORKAROUNDS).
+        key = self.make_key(AuthenticatedKey)
+        payload = b"payload"
+        id = key.id_hash(payload)
+        envelope = bytearray(key.encrypt(id, payload, aad=b"aad"))
+        envelope[5] ^= 1  # corrupt the tag
+        monkeypatch.setattr("borg.crypto.key.AUTHENTICATED_NO_KEY", True)
+        assert key.decrypt(id, bytes(envelope), aad=b"aad") == payload
+
+    def test_unkeyed_modes_verify_despite_workaround(self, monkeypatch):
+        # the unkeyed modes need no key material, so the workaround must not switch their
+        # checksum verification off.
+        monkeypatch.setattr("borg.crypto.key.AUTHENTICATED_NO_KEY", True)
+        key = self.make_key(ChecksumKey)
+        payload = b"payload"
+        id = key.id_hash(payload)
+        envelope = bytearray(key.encrypt(id, payload, aad=b"aad"))
+        envelope[5] ^= 1
+        with pytest.raises(IntegrityError):
+            key.decrypt(id, bytes(envelope), aad=b"aad")
+
+
+def test_dropped_borg2_beta_key_types(tmpdir):
+    # the borg2 beta "none"/"authenticated" formats were dropped, see #9104. A borg2 repository
+    # using them must be refused instead of being read with the legacy classes.
+    from ...repoobj import RepoObj
+    from ...crypto.key import key_factory
+
+    for legacy_cls in (LegacyPlaintextKey, LegacyAuthenticatedKey):
+        key = legacy_cls(MagicMock(id=bytes(32)))
+        if legacy_cls is LegacyAuthenticatedKey:
+            key.id_key = bytes(32)
+        manifest_chunk = RepoObj(key).format(bytes(32), {}, b"manifest", ro_type=ROBJ_MANIFEST)
+        with pytest.raises(UnsupportedPayloadError):
+            key_factory(MagicMock(id=bytes(32)), manifest_chunk, ro_cls=RepoObj)
+
+
+def test_dropped_blake3_authenticated_type_byte():
+    with pytest.raises(UnsupportedPayloadError):
+        identify_key(bytes([KeyType.DROPPED_BLAKE3AUTHENTICATED]) + b"payload")
+
+
 def test_decrypt_key_file_unsupported_algorithm():
     """We will add more algorithms in the future. We should raise a helpful error."""
     key = CHPOKey(None)
diff --git a/src/borg/testsuite/legacy_archives_test.py b/src/borg/testsuite/legacy_archives_test.py
index 916b709f2d..29d8aa6617 100644
--- a/src/borg/testsuite/legacy_archives_test.py
+++ b/src/borg/testsuite/legacy_archives_test.py
@@ -6,7 +6,7 @@
 
 import pytest
 
-from ..crypto.key import PlaintextKey
+from ..legacy.crypto.key import PlaintextKey
 from ..helpers.errors import CommandError, Error
 from ..legacy.archives import LegacyArchives
 from ..legacy.repository import LegacyRepository
diff --git a/src/borg/testsuite/repoobj_test.py b/src/borg/testsuite/repoobj_test.py
index 4b41bacf2a..db98ff6b6c 100644
--- a/src/borg/testsuite/repoobj_test.py
+++ b/src/borg/testsuite/repoobj_test.py
@@ -1,7 +1,7 @@
 import pytest
 
 from ..constants import ROBJ_FILE_STREAM, ROBJ_MANIFEST, ROBJ_ARCHIVE_META
-from ..crypto.key import PlaintextKey, AuthenticatedKey, CHPOKey
+from ..crypto.key import ChecksumKey, AuthenticatedKey, CHPOKey, LegacyPlaintextKey
 from ..helpers import msgpack
 from ..helpers.errors import Error, IntegrityError
 from ..repository import Repository
@@ -26,12 +26,19 @@ def repository(tmpdir):
 
 @pytest.fixture
 def key(repository):
-    return PlaintextKey(repository)
+    # "none-sha256" mode: not encrypted and not authenticated, but checksummed.
+    return ChecksumKey(repository)
+
+
+@pytest.fixture
+def legacy_key(repository):
+    # borg 1.x "none" mode: no envelope protection at all (read-only in borg 2).
+    return LegacyPlaintextKey(repository)
 
 
 @pytest.fixture
 def authenticated_key(repository):
-    # "authenticated" mode: not encrypted, the id check is the only read path authentication.
+    # "authenticated-sha256" mode: not encrypted, but the envelope is MAC-authenticated.
     key = AuthenticatedKey(repository)
     key.init_from_random_data()
     key.init_ciphers()
@@ -40,7 +47,6 @@ def authenticated_key(repository):
 
 @pytest.fixture
 def aead_key(repository):
-    # AEAD key, needed to test header_aad authentication; PlaintextKey is unauthenticated.
     key = CHPOKey(repository)
     key.init_from_random_data()
     key.init_ciphers()
@@ -70,8 +76,8 @@ def test_format_parse_roundtrip(key):
     assert edata.startswith(bytes((key.TYPE,)))
 
 
-def test_format_parse_roundtrip_borg1(key):  # legacy
-    repo_objs = RepoObj1(key)
+def test_format_parse_roundtrip_borg1(legacy_key):  # legacy
+    repo_objs = RepoObj1(legacy_key)
     data = b"foobar" * 10
     id = repo_objs.id_hash(data)
     meta = {}  # borg1 does not support this kind of metadata
@@ -90,13 +96,14 @@ def test_format_parse_roundtrip_borg1(key):  # legacy
     assert edata.startswith(bytes((key.TYPE, compressor.ID, compressor.level)))
 
 
-def test_borg1_borg2_transition(key):
+def test_borg1_borg2_transition(legacy_key, key):
     # Borg transfer reads Borg 1.x repository objects (without decompressing them),
     # and writes Borg 2 repository objects (providing already-compressed data to avoid recompression).
+    # The borg 1.x "none" and the borg 2 "none-sha256" mode use the same (unkeyed sha256) chunk ids.
     meta = {}  # borg1 does not support this kind of metadata
     data = b"foobar" * 10
     len_data = len(data)
-    repo_objs1 = RepoObj1(key)
+    repo_objs1 = RepoObj1(legacy_key)
     id = repo_objs1.id_hash(data)
     borg1_cdata = repo_objs1.format(id, meta, data, ro_type=ROBJ_FILE_STREAM)
     meta1, compr_data1 = repo_objs1.parse(
@@ -196,16 +203,23 @@ def _tamper(cdata, offset):
     return bytes(tampered)
 
 
-def test_tampered_header_chunk_id_detected(aead_key):
-    # chunk_id is part of header_aad, so tampering with it fails AEAD authentication in
+@pytest.fixture(params=["key", "authenticated_key", "aead_key"])
+def protected_key(request):
+    # every borg 2 mode protects the object with a tag over the payload and the AAD - an unkeyed
+    # checksum for "none-*", a MAC for "authenticated-*", the AEAD tag for the encrypted modes.
+    return request.getfixturevalue(request.param)
+
+
+def test_tampered_header_chunk_id_detected(protected_key):
+    # chunk_id is part of header_aad, so tampering with it fails the tag verification in
     # parse()/parse_meta().
-    repo_objs = RepoObj(aead_key)
+    repo_objs = RepoObj(protected_key)
     data = b"foobar" * 10
     id = repo_objs.id_hash(data)
     cdata = repo_objs.format(id, {"custom": "something"}, data, ro_type=ROBJ_FILE_STREAM)
 
     # chunk_id is at header offset 9..41 (after 8-byte magic + 1-byte version). It has no structural
-    # check, so tampering is detected only through AEAD authentication.
+    # check, so tampering is detected only through the tag.
     tampered = _tamper(cdata, offset=9)
     with pytest.raises(IntegrityError):
         repo_objs.parse_meta(id, tampered, ro_type=ROBJ_FILE_STREAM)
@@ -213,11 +227,11 @@ def test_tampered_header_chunk_id_detected(aead_key):
         repo_objs.parse(id, tampered, ro_type=ROBJ_FILE_STREAM)
 
 
-def test_tampered_header_magic_detected(aead_key):
+def test_tampered_header_magic_detected(protected_key):
     # A tampered magic byte is rejected by the structural check (`hdr.magic != OBJ_MAGIC`) before
-    # key.decrypt() runs, so this does not test AEAD authentication of header_aad - see
+    # key.decrypt() runs, so this does not test the authentication of header_aad - see
     # test_header_aad_tamper_detected_at_key_layer for that.
-    repo_objs = RepoObj(aead_key)
+    repo_objs = RepoObj(protected_key)
     data = b"foobar" * 10
     id = repo_objs.id_hash(data)
     cdata = repo_objs.format(id, {"custom": "something"}, data, ro_type=ROBJ_FILE_STREAM)
@@ -230,34 +244,90 @@ def test_tampered_header_magic_detected(aead_key):
         repo_objs.parse(id, tampered, ro_type=ROBJ_FILE_STREAM)
 
 
-def test_header_aad_tamper_detected_at_key_layer(aead_key):
+def test_tampered_meta_detected(protected_key):
+    # The metadata (which selects the decompressor and the payload size!) is covered by the tag in
+    # every mode, so it can not be modified without the read failing - and the check happens before
+    # the metadata is used for anything, see RepoObj.parse.
+    repo_objs = RepoObj(protected_key)
+    data = b"foobar" * 10
+    id = repo_objs.id_hash(data)
+    cdata = repo_objs.format(id, {"custom": "something"}, data, ro_type=ROBJ_FILE_STREAM)
+
+    hdr_size = RepoObj.obj_header.size
+    hdr = RepoObj.ObjHeader(*RepoObj.obj_header.unpack(cdata[:hdr_size]))
+    for offset in range(hdr_size, hdr_size + hdr.meta_size):  # every byte of the meta slot
+        tampered = _tamper(cdata, offset=offset)
+        with pytest.raises(IntegrityError):
+            repo_objs.parse_meta(id, tampered, ro_type=ROBJ_FILE_STREAM)
+        with pytest.raises(IntegrityError):
+            repo_objs.parse(id, tampered, ro_type=ROBJ_FILE_STREAM)
+
+
+def test_tampered_data_detected(protected_key):
+    repo_objs = RepoObj(protected_key)
+    data = b"foobar" * 10
+    id = repo_objs.id_hash(data)
+    cdata = repo_objs.format(id, {"custom": "something"}, data, ro_type=ROBJ_FILE_STREAM)
+
+    hdr_size = RepoObj.obj_header.size
+    hdr = RepoObj.ObjHeader(*RepoObj.obj_header.unpack(cdata[:hdr_size]))
+    for offset in range(hdr_size + hdr.meta_size, len(cdata)):  # every byte of the data slot
+        tampered = _tamper(cdata, offset=offset)
+        with pytest.raises(IntegrityError):
+            repo_objs.parse(id, tampered, ro_type=ROBJ_FILE_STREAM)
+
+
+def test_checksum_mode_does_not_authenticate(key):
+    # The honest limit of the "none-*" modes: their tag is an unkeyed checksum, so somebody who
+    # modifies an object can just recompute it. Detecting that needs a secret - the
+    # "authenticated-*" and the encrypted modes have one, this mode does not.
+    repo_objs = RepoObj(key)
+    data = b"foobar" * 10
+    id = repo_objs.id_hash(data)
+    cdata = repo_objs.format(id, {"custom": "something"}, data, ro_type=ROBJ_FILE_STREAM)
+
+    hdr_size = RepoObj.obj_header.size
+    hdr = RepoObj.ObjHeader(*RepoObj.obj_header.unpack(cdata[:hdr_size]))
+    header_aad = OBJ_MAGIC + bytes([OBJ_VERSION]) + id
+    # rewrite the meta slot with attacker-chosen content and a recomputed checksum
+    forged_meta = dict(repo_objs.parse_meta(id, cdata, ro_type=ROBJ_FILE_STREAM), custom="forged")
+    forged_meta_slot = key.encrypt(id, msgpack.packb(forged_meta), aad=header_aad + b"M")
+    forged = (
+        RepoObj.obj_header.pack(hdr.magic, hdr.version, hdr.chunk_id, len(forged_meta_slot), hdr.data_size)
+        + forged_meta_slot
+        + cdata[hdr_size + hdr.meta_size :]
+    )
+    assert repo_objs.parse_meta(id, forged, ro_type=ROBJ_FILE_STREAM)["custom"] == "forged"
+
+
+def test_header_aad_tamper_detected_at_key_layer(protected_key):
     # Calls key.encrypt()/key.decrypt() directly with header_aad, to check that every byte of
     # header_aad (magic, version, chunk_id) is authenticated, not just chunk_id.
     data = b"foobar" * 10
-    id = aead_key.id_hash(data)
+    id = protected_key.id_hash(data)
     header_aad = OBJ_MAGIC + bytes([OBJ_VERSION]) + id
-    encrypted = aead_key.encrypt(id, data, aad=header_aad)
+    encrypted = protected_key.encrypt(id, data, aad=header_aad)
 
-    assert aead_key.decrypt(id, encrypted, aad=header_aad) == data
+    assert protected_key.decrypt(id, encrypted, aad=header_aad) == data
 
     # tamper the magic byte (offset 0) after encryption; decrypt gets a different header_aad than encrypt did.
     tampered_header_aad = bytearray(header_aad)
     tampered_header_aad[0] ^= 0x01
     with pytest.raises(IntegrityError):
-        aead_key.decrypt(id, encrypted, aad=bytes(tampered_header_aad))
+        protected_key.decrypt(id, encrypted, aad=bytes(tampered_header_aad))
 
     # tamper the version byte (offset 8) after encryption.
     tampered_header_aad = bytearray(header_aad)
     tampered_header_aad[8] ^= 0x01
     with pytest.raises(IntegrityError):
-        aead_key.decrypt(id, encrypted, aad=bytes(tampered_header_aad))
+        protected_key.decrypt(id, encrypted, aad=bytes(tampered_header_aad))
 
 
-def test_meta_data_slot_swap_detected(aead_key):
-    # meta_encrypted and data_encrypted carry different slot tags in their AAD, so splicing one
+def test_meta_data_slot_swap_detected(protected_key):
+    # the meta and the data slot carry different slot tags in their AAD, so splicing one
     # into the other's position (fixing up meta_size/data_size to match the swapped lengths) must
-    # fail authentication instead of silently decrypting under the wrong slot.
-    repo_objs = RepoObj(aead_key)
+    # fail the tag verification instead of silently being read under the wrong slot.
+    repo_objs = RepoObj(protected_key)
     data = b"foobar" * 10
     id = repo_objs.id_hash(data)
     cdata = repo_objs.format(id, {"custom": "something"}, data, ro_type=ROBJ_FILE_STREAM)
@@ -378,11 +448,10 @@ def test_assert_id_places_env_var_mandatory_place(aead_key, monkeypatch):
     assert "verify_data: always verifies, can not be configured" in str(exc_info.value)
 
 
-@pytest.mark.parametrize("key_fixture", ["key", "authenticated_key"])
-def test_assert_id_never_skipped_for_non_aead_key(key_fixture, request, monkeypatch):
-    # PlaintextKey ("none" mode) and AuthenticatedKey ("authenticated" mode): there is no AEAD, so the id
-    # check is the only integrity check reads have and thus must happen no matter what the user configured.
-    key = request.getfixturevalue(key_fixture)
+def test_assert_id_never_skipped_for_unauthenticated_key(key, monkeypatch):
+    # ChecksumKey ("none-sha256" mode): the envelope checksum is unkeyed and thus no authentication,
+    # so the id check is the only integrity check reads have and must happen no matter what the
+    # user configured.
     monkeypatch.setenv("BORG_ASSERT_ID", "")  # verify nowhere - but this is not switchable off
     assert key.id_check_is_authentication
     repo_objs = RepoObj(key)
@@ -394,6 +463,22 @@ def test_assert_id_never_skipped_for_non_aead_key(key_fixture, request, monkeypa
             repo_objs.parse(id, cdata, ro_type=ROBJ_FILE_STREAM, assert_id_place=place)
 
 
+def test_assert_id_configurable_for_authenticated_key(authenticated_key, monkeypatch):
+    # AuthenticatedKey ("authenticated-sha256"): the envelope MAC authenticates every read (the
+    # chunk id is in the AAD), so verifying the id on top of that is configurable, like for AEAD.
+    monkeypatch.setenv("BORG_ASSERT_ID", "")  # verify at none of the configurable places
+    assert not authenticated_key.id_check_is_authentication
+    repo_objs = RepoObj(authenticated_key)
+    id = repo_objs.id_hash(b"foobar" * 10)
+    cdata = wrong_content_object(repo_objs, id)
+
+    for place in ASSERT_ID_PLACES:
+        repo_objs.parse(id, cdata, ro_type=ROBJ_FILE_STREAM, assert_id_place=place)
+    for place in ASSERT_ID_PLACES_MANDATORY:  # ... but check --verify-data always verifies
+        with pytest.raises(IntegrityError):
+            repo_objs.parse(id, cdata, ro_type=ROBJ_FILE_STREAM, assert_id_place=place)
+
+
 def test_version1_object_without_header_aad_still_readable(aead_key):
     # Builds an OBJ_VERSION_NO_HEADER_AAD object by hand (format() only writes OBJ_VERSION_HEADER_AAD)
     # and checks that parse()/parse_meta() still decrypt it.

From 9f07fd80577e9dec863a88d3dc44875418a8c943 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 13 Aug 2026 23:03:14 +0200
Subject: [PATCH 116/144] with-lock: fix epilog about leftover lock
 auto-removal, see #9870

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/man/borg-with-lock.1      | 8 ++++----
 docs/usage/with-lock.rst.inc   | 6 +++---
 src/borg/archiver/lock_cmds.py | 6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1
index 511e306e04..4e8c87f517 100644
--- a/docs/man/borg-with-lock.1
+++ b/docs/man/borg-with-lock.1
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "borg-with-lock" "1" "2026-07-21" "" "borg backup tool"
+.TH "borg-with-lock" "1" "2026-08-13" "" "borg backup tool"
 .SH Name
 borg-with-lock \- Runs a user-specified command with the repository lock held.
 .SH SYNOPSIS
@@ -55,9 +55,9 @@ code as Borg\(aqs return code.
 .INDENT 0.0
 .INDENT 3.5
 If you copy a repository with the lock held, the lock will be present in
-the copy. Before using Borg on the copy from a different host,
-you need to run \fBborg break\-lock\fP on the copied repository, because
-Borg is cautious and does not automatically remove stale locks made by a different host.
+the copy. Borg removes such a leftover lock automatically, but only after
+a safety period of about half an hour, so to use Borg on the copy right
+away, first run \fBborg break\-lock\fP on the copied repository.
 .UNINDENT
 .UNINDENT
 .SH OPTIONS
diff --git a/docs/usage/with-lock.rst.inc b/docs/usage/with-lock.rst.inc
index 8e8db70bbd..fa5f148e5f 100644
--- a/docs/usage/with-lock.rst.inc
+++ b/docs/usage/with-lock.rst.inc
@@ -60,6 +60,6 @@ code as Borg's return code.
 .. note::
 
     If you copy a repository with the lock held, the lock will be present in
-    the copy. Before using Borg on the copy from a different host,
-    you need to run ``borg break-lock`` on the copied repository, because
-    Borg is cautious and does not automatically remove stale locks made by a different host.
\ No newline at end of file
+    the copy. Borg removes such a leftover lock automatically, but only after
+    a safety period of about half an hour, so to use Borg on the copy right
+    away, first run ``borg break-lock`` on the copied repository.
\ No newline at end of file
diff --git a/src/borg/archiver/lock_cmds.py b/src/borg/archiver/lock_cmds.py
index 10c2751e1f..209535be0d 100644
--- a/src/borg/archiver/lock_cmds.py
+++ b/src/borg/archiver/lock_cmds.py
@@ -67,9 +67,9 @@ def build_parser_locks(self, subparsers, common_parser, mid_common_parser):
         .. note::
 
             If you copy a repository with the lock held, the lock will be present in
-            the copy. Before using Borg on the copy from a different host,
-            you need to run ``borg break-lock`` on the copied repository, because
-            Borg is cautious and does not automatically remove stale locks made by a different host.
+            the copy. Borg removes such a leftover lock automatically, but only after
+            a safety period of about half an hour, so to use Borg on the copy right
+            away, first run ``borg break-lock`` on the copied repository.
         """
         )
         subparser = ArgumentParser(

From e15404aa2f3a7ef807eb411d3673e8ff17e0aeff Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 12:24:54 +0200
Subject: [PATCH 117/144] locking: ignore locks vanishing between listing and
 loading them

Two clients racing on stale-lock cleanup could crash with an uncaught
borgstore ObjectNotFound traceback: one client deletes a stale lock while
the other is between listing the locks directory and loading that lock
object. Skip such locks - semantically they are the same as a lock that
was already gone at listing time.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/storelocking.py                |  7 ++++++-
 src/borg/testsuite/storelocking_test.py | 20 +++++++++++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/src/borg/storelocking.py b/src/borg/storelocking.py
index f781862383..c6b90d89c5 100644
--- a/src/borg/storelocking.py
+++ b/src/borg/storelocking.py
@@ -150,7 +150,12 @@ def _get_locks(self):
             return {}
         for info in infos:
             key = info.name
-            content = self.store.load(f"locks/{key}")
+            try:
+                content = self.store.load(f"locks/{key}")
+            except ObjectNotFound:
+                # the lock vanished between our listing and loading it, e.g. it was released
+                # by its owner or another client killed it as stale - so just ignore it.
+                continue
             lock = json.loads(content.decode("utf-8"))
             lock["key"] = key
             lock["dt"] = datetime.datetime.fromisoformat(lock["time"])
diff --git a/src/borg/testsuite/storelocking_test.py b/src/borg/testsuite/storelocking_test.py
index a5cb7345f9..cd05af0100 100644
--- a/src/borg/testsuite/storelocking_test.py
+++ b/src/borg/testsuite/storelocking_test.py
@@ -3,7 +3,7 @@
 
 import pytest
 
-from borgstore.store import Store
+from borgstore.store import ObjectNotFound, Store
 
 from ..storelocking import Lock, NotLocked, LockTimeout
 
@@ -156,6 +156,24 @@ def delete(name, *args, **kwargs):
             lock.refresh()
         assert len(list(lockstore.list("locks"))) == 0  # no new lock left behind
 
+    def test_lock_vanished_between_list_and_load(self, lockstore, monkeypatch):
+        # another client can delete a lock (e.g. kill it as stale, or its owner releases it)
+        # between our listing and our loading of it - such a lock must be skipped, not crash us.
+        foreign_key = Lock(lockstore, id=ID1)._create_lock(exclusive=True)
+
+        orig_load = lockstore.load
+
+        def load_vanished(name, *args, **kwargs):
+            if name == f"locks/{foreign_key}":
+                raise ObjectNotFound(name)
+            return orig_load(name, *args, **kwargs)
+
+        monkeypatch.setattr(lockstore, "load", load_vanished)
+        lock = Lock(lockstore, exclusive=True, id=ID2)
+        assert foreign_key not in lock._get_locks()  # skipped, no exception
+        lock.acquire()  # the vanished exclusive lock must not block us
+        lock.release()
+
     def test_migrate_lock(self, lockstore):
         old_id, new_id = ID1, ID2
         assert old_id[1] != new_id[1]  # different PIDs (like when doing daemonize())

From 8b4e98a3534c560f8d241d9aa1daf2b9d942a340 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 12:45:44 +0200
Subject: [PATCH 118/144] CI: daily canary probing Python 3.12 on ubuntu-26.04

ubuntu-26.04 runners only work with the image's default Python (3.14);
actions/setup-python has no cached builds of older Pythons for that
image yet. This scheduled probe tries setup-python 3.12 daily with
inverted semantics: green while 3.12 is still unavailable, red the day
it starts working. GitHub emails scheduled-workflow failures to the
user who last modified the cron line, so the red run doubles as the
notification to enable 3.12 in ci.yml and remove this canary.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/canary-py312-ubuntu2604.yml | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 .github/workflows/canary-py312-ubuntu2604.yml

diff --git a/.github/workflows/canary-py312-ubuntu2604.yml b/.github/workflows/canary-py312-ubuntu2604.yml
new file mode 100644
index 0000000000..a770915ad8
--- /dev/null
+++ b/.github/workflows/canary-py312-ubuntu2604.yml
@@ -0,0 +1,49 @@
+# Inverted canary: watch for Python 3.12 becoming available on ubuntu-26.04.
+#
+# ubuntu-26.04 runners currently only work with the image's default Python
+# (3.14) - actions/setup-python has no cached builds of older Pythons for
+# this image yet, so requesting 3.12 fails. This probe tries daily.
+#
+# The logic is inverted on purpose: while 3.12 is still unavailable, the
+# run stays GREEN (expected state, no noise). The day setup-python 3.12
+# succeeds, the job FAILS loudly - GitHub emails scheduled-workflow
+# failures to the user who last modified the cron line, so a red run here
+# is the "3.12 works now" notification. When it fires: add ubuntu-26.04
+# / 3.12 jobs to ci.yml and delete this workflow.
+
+name: Canary (Python 3.12 on ubuntu-26.04)
+
+on:
+  schedule:
+    - cron: '0 6 * * *'  # Run at 06:00 UTC
+  workflow_dispatch:      # Allow manual trigger
+
+permissions: {}
+
+jobs:
+  probe:
+    name: Probe setup-python 3.12
+    runs-on: ubuntu-26.04
+    timeout-minutes: 30
+
+    steps:
+    - name: Try to set up Python 3.12
+      id: setup
+      continue-on-error: true
+      uses: actions/setup-python@v7
+      with:
+        python-version: '3.12'
+
+    - name: Still unsupported - stay green
+      if: steps.setup.outcome == 'failure'
+      run: |
+        echo "::notice::Python 3.12 is still unavailable on ubuntu-26.04 (expected - nothing to do)."
+
+    - name: Python 3.12 works now - fail to trigger the notification email
+      if: steps.setup.outcome == 'success'
+      run: |
+        python --version
+        python -c 'import ssl, sqlite3, zlib, lzma'
+        python -m pip --version
+        echo "::error::Python 3.12 is NOW AVAILABLE on ubuntu-26.04! Add it to ci.yml and delete this canary workflow. (This failure is the notification - see the comment at the top of the workflow file.)"
+        exit 1

From b8546e9a8f0a12ac6e4a7309c0108bca93cece41 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 13:29:04 +0200
Subject: [PATCH 119/144] docs: add generated man page borg-environment.1

PR #10063 added the "borg help environment" topic and build_man
generates borg-environment.1 since then, but the generated man page
was never committed. Add it (built via scripts/make.py build_man).
---
 docs/man/borg-environment.1 | 612 ++++++++++++++++++++++++++++++++++++
 1 file changed, 612 insertions(+)
 create mode 100644 docs/man/borg-environment.1

diff --git a/docs/man/borg-environment.1 b/docs/man/borg-environment.1
new file mode 100644
index 0000000000..c7f45c7db5
--- /dev/null
+++ b/docs/man/borg-environment.1
@@ -0,0 +1,612 @@
+.\" Man page generated from reStructuredText
+.\" by the Docutils 0.22.4 manpage writer.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "borg-environment" "1" "2026-08-14" "" "borg backup tool"
+.SH Name
+borg-environment \- Details regarding environment
+.SH DESCRIPTION
+.sp
+Borg uses some environment variables for automation:
+.INDENT 0.0
+.TP
+.B General:
+.INDENT 7.0
+.TP
+.B BORG_REPO
+When set, use the value to give the default repository location.
+Use this so you do not need to type \fB\-\-repo /path/to/my/repo\fP all the time.
+.TP
+.B BORG_OTHER_REPO
+Similar to BORG_REPO, but gives the default for \fB\-\-other\-repo\fP\&.
+.TP
+.B BORG_PASSPHRASE (and BORG_OTHER_PASSPHRASE)
+When set, use the value to answer the passphrase question for encrypted repositories.
+It is used when a passphrase is needed to access an encrypted repo as well as when a new
+passphrase should be initially set when initializing an encrypted repo.
+See also BORG_NEW_PASSPHRASE.
+.TP
+.B BORG_PASSCOMMAND (and BORG_OTHER_PASSCOMMAND)
+When set, use the standard output of the command (trailing newlines are stripped) to answer the
+passphrase question for encrypted repositories.
+It is used when a passphrase is needed to access an encrypted repo as well as when a new
+passphrase should be initially set when initializing an encrypted repo. Note that the command
+is executed without a shell. So variables, like \fB$HOME\fP will work, but \fB~\fP won\(aqt.
+If BORG_PASSPHRASE is also set, it takes precedence.
+See also BORG_NEW_PASSPHRASE.
+.TP
+.B BORG_PASSPHRASE_FD (and BORG_OTHER_PASSPHRASE_FD)
+When set, specifies a file descriptor to read a passphrase
+from. Programs starting borg may choose to open an anonymous pipe
+and use it to pass a passphrase. This is safer than passing via
+BORG_PASSPHRASE, because on some systems (e.g. Linux) environment
+can be examined by other processes.
+If BORG_PASSPHRASE or BORG_PASSCOMMAND are also set, they take precedence.
+.TP
+.B BORG_NEW_PASSPHRASE
+When set, use the value to answer the passphrase question when a \fBnew\fP passphrase is asked for.
+This variable is checked first. If it is not set, BORG_PASSPHRASE and BORG_PASSCOMMAND will also
+be checked.
+Main use case for this is to fully automate \fBborg key change\-passphrase\fP\&.
+.TP
+.B BORG_DISPLAY_PASSPHRASE
+When set, use the value to answer the \(dqdisplay the passphrase for verification\(dq question when defining a new passphrase for encrypted repositories.
+.TP
+.B BORG_DEBUG_PASSPHRASE
+When set to YES, display debugging information that includes passphrases used and passphrase related env vars set.
+.TP
+.B BORG_EXIT_CODES
+When set to \(dqmodern\(dq, the borg process will return more specific exit codes (rc).
+When set to \(dqlegacy\(dq, the borg process will return rc 2 for all errors, 1 for all warnings, 0 for success.
+Default is \(dqmodern\(dq.
+.TP
+.B BORG_HOST_ID
+Borg usually computes a host id from the FQDN plus the results of \fBuuid.getnode()\fP (which usually returns
+a unique id based on the MAC address of the network interface. Except if that MAC happens to be all\-zero \- in
+that case it returns a random value, which is not what we want (because it kills automatic stale lock removal).
+So, if you have an all\-zero MAC address or other reasons to better control the host id externally, just set this
+environment variable to a unique value. If all your FQDNs are unique, you can just use the FQDN. If not,
+use \%<FQDN@\:uniqueid>\&.
+.TP
+.B BORG_HOSTNAME
+When set, use this value as the hostname (instead of the auto\-detected one), e.g. to run borg
+on one host, but impersonate another host. This affects the hostname stored in newly created
+archives as well as the \fB{hostname}\fP placeholder.
+.TP
+.B BORG_USERNAME
+When set, use this value as the username (instead of the auto\-detected one), e.g. to run borg
+as one user, but impersonate another user. This affects the username stored in newly created
+archives as well as the \fB{user}\fP placeholder.
+.TP
+.B BORG_LOCK_WAIT
+You can set the default value for the \fB\-\-lock\-wait\fP option with this, so
+you do not need to give it as a command line option.
+.TP
+.B BORG_LOGGING_CONF
+When set, use the given filename as INI\-style logging configuration (see
+\%<https://\:docs\:.python\:.org/\:3/\:library/\:logging\:.config\:.html#\:configuration-file-format>).
+A basic example conf can be found at \fBdocs/misc/logging.conf\fP\&.
+.TP
+.B BORG_RSH
+When set, use this command instead of \fBssh\fP\&. This can be used to specify ssh options, such as
+a custom identity file \fBssh \-i /path/to/private/key\fP\&. See \fBman ssh\fP for other options.
+This is the replacement for the removed \fB\-\-rsh CMD\fP command line option.
+borg also gives this to borgstore as \fBBORGSTORE_RSH\fP, except if that is already set.
+.TP
+.B BORG_REMOTE_PATH
+When set, use the given path as borg executable on the remote (defaults to \(dqborg\(dq if unset).
+This is the replacement for the removed \fB\-\-remote\-path PATH\fP command line option.
+.TP
+.B BORG_UNITS
+Determines how borg formats sizes in its human\-readable output:
+.INDENT 7.0
+.IP \(bu 2
+\fBsi\fP (default): decimal units, e.g. \fB1.23 MB\fP (1kB = 1000B)
+.IP \(bu 2
+\fBiec\fP: binary units, e.g. \fB1.18 MiB\fP (1KiB = 1024B)
+.IP \(bu 2
+\fBraw\fP: exact byte counts, e.g. \fB1234567 B\fP
+.UNINDENT
+.sp
+Use \fBraw\fP if you want to parse sizes with scripts (e.g. for monitoring),
+so you do not have to deal with scaled values and different units.
+Alternatively, use a command\(aqs \fB\-\-json\fP output or, for the commands
+supporting \fB\-\-format\fP, the size related format keys \- sizes are given
+as byte counts there anyway.
+.sp
+\fBBORG_UNITS=iec\fP is the replacement for the removed \fBBORG_IEC\fP environment
+variable (and for the \fB\-\-iec\fP command line option removed before that).
+.TP
+.B BORG_PROGRESS_FPS
+How often the \fB\-\-progress\fP output is updated at most, in updates per
+second (default: 5). Fractional values are allowed, e.g.
+\fBBORG_PROGRESS_FPS=0.1\fP limits it to one update every 10 seconds.
+Lower values are useful when the output goes into a logfile rather than
+to an interactive terminal.
+.TP
+.B BORG_SPINNER
+Controls the spinner borg animates on a terminal while doing work of unknown
+duration:
+.INDENT 7.0
+.IP \(bu 2
+unset (default): animate, using Unicode frames if the terminal can display them
+.IP \(bu 2
+\fBascii\fP: animate, but only use ASCII frames (\fB|/\-\e\fP)
+.IP \(bu 2
+\fBoff\fP: do not animate, only output the messages next to the spinner
+.UNINDENT
+.sp
+The spinner is animated only on an interactive terminal anyway (and never
+with \fB\-\-log\-json\fP), and its colour follows the usual \fBNO_COLOR\fP and
+\fBCOLORTERM\fP conventions. See also \fBBORG_PROGRESS_FPS\fP: it also gives
+the spinner its frame rate.
+.TP
+.B BORG_DEBUG_PROFILE
+When set to a filename, write an execution profile in Borg format into that file
+(see \fIdebugging\fP). If the filename ends with \fB\&.pyprof\fP, a Python\-compatible
+profile is written instead.
+This is the replacement for the removed \fB\-\-debug\-profile\fP command line option.
+Note: every borg invocation writes the profile, so unset it again when you are done.
+.TP
+.B BORG_REPO_PERMISSIONS
+Set repository permissions, see also: \fIborg_serve\fP
+.TP
+.B BORG_FILES_CACHE_SUFFIX
+When set to a value at least one character long, instructs borg to use a specifically named
+(based on the suffix) alternative files cache. This can be used to avoid loading and saving
+cache entries for backup sources other than the current sources.
+.TP
+.B BORG_FILES_CACHE_TTL
+When set to a numeric value, this determines the maximum \(dqtime to live\(dq for the files cache
+entries (default: 2). The files cache is used to determine quickly whether a file is unchanged.
+.TP
+.B BORG_ASSERT_ID
+Comma\-separated list of the places where borg shall verify that a chunk\(aqs content matches
+its chunk id (\fBchunkid == id_hash(content)\fP) after decrypting and decompressing it.
+Verifying costs a full hash pass over everything that is read at such a place.
+.sp
+Default (variable not set):
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.EX
+BORG_ASSERT_ID=repair,transfer,rechunk
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+These are the place names that can be listed:
+.INDENT 7.0
+.TP
+.B read
+Every read that decompresses a chunk: \fBborg extract\fP, \fBborg mount\fP,
+\fBborg export\-tar\fP, \fBborg diff\fP, ... This is by far the most data borg reads, so
+this place is \fBnot\fP in the default, see the explanation below.
+.TP
+.B repair
+\fBborg check \-\-repair\fP\&. It rebuilds archives from the item metadata stream it reads,
+re\-packing it into new chunks with freshly computed ids, and it recreates manifest and
+archives directory entries from what it reads.
+.TP
+.B transfer
+\fBborg transfer\fP, for everything it reads from the source repository. Transferring
+re\-anchors the content in another repository, which is a trust boundary.
+.TP
+.B rechunk
+\fBborg recreate \-\-chunker\-params ...\fP, i.e. re\-chunking reads. Re\-chunking computes
+new chunk ids from the content it reads, so a violation would not be noticeable any
+more afterwards. (Re\-chunking in \fBborg transfer\fP is covered by \fBtransfer\fP\&.)
+.UNINDENT
+.sp
+An unknown place name is an error. An empty value (\fBBORG_ASSERT_ID=\fP) verifies at none of
+these places, but still where borg always verifies (see below).
+.sp
+Why \fBread\fP is not in the default: for encrypted repositories (all the AEAD ciphersuites),
+the chunk id is part of the AEAD additional authenticated data, so a successful decryption
+already proves that a holder of the repository key deliberately stored exactly this
+ciphertext for exactly this chunk id. A malicious or buggy \fBrepository\fP can therefore not
+swap, splice or substitute objects, whether the id is verified on read or not. What the id
+check adds is the detection of chunks whose content does not match their id, which only a
+malicious or compromised \fBborg client that had your borg key\fP could have written (e.g. to
+poison future deduplication). If that is in your threat model \- e.g. because some machines
+writing into the repository are not fully trusted \- add \fBread\fP to the list:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.EX
+BORG_ASSERT_ID=read,repair,transfer,rechunk
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+Otherwise, running \fBborg check \-\-verify\-data\fP periodically is recommended: it is the
+audit that re\-certifies the invariant for all chunks in the background, instead of on
+every read.
+.sp
+Independent of this variable, borg always verifies the chunk id:
+.INDENT 7.0
+.IP \(bu 2
+in \fBborg check \-\-verify\-data\fP\&. That audit is what makes not verifying elsewhere
+defensible, so it is not configurable (there is no \fBverify_data\fP place name).
+.IP \(bu 2
+for \fBauthenticated\fP and \fBnone\fP mode repositories: there is no AEAD there, so the id
+check \fIis\fP the read path\(aqs integrity check and switching it off would remove it
+completely. Same for reading borg 1.x repositories (\fBborg transfer\fP).
+.UNINDENT
+.TP
+.B BORG_BLAKE3_MT_THRESHOLD
+When set to a numeric value, chunks of at least that many KiB get their id computed by
+multi\-threaded BLAKE3, smaller ones single\-threaded (default: 256, i.e. 256KiB).
+Only relevant for repositories using \fB\-\-id\-hash blake3\fP\&.
+Multi\-threading only pays off for big enough chunks and the break\-even point depends on
+the machine\(aqs core count, so the default is deliberately conservative.
+Run \fBscripts/blake3\-optimize\-mt\-threshold.py\fP to measure the best value for your
+machine \- it sweeps input sizes, prints the recommended threshold and the command to
+set it, and can optionally show a chart of the measurements in your browser
+(\fB\-\-html \-\-open\fP).
+0 means \(dqalways multi\-threaded\(dq, a very large value effectively disables multi\-threading.
+.TP
+.B BORG_ZSTD_MT_WORKERS
+When set to a numeric value, use that many threads to zstd\-compress a single chunk
+(default: the cpu count). 0 or 1 means single\-threaded compression.
+Only relevant when compressing with \fBzstd\fP\&.
+Chunks below 768KiB are always compressed single\-threaded: libzstd will not use a
+compression job smaller than 512KiB, so a small chunk gets split very unevenly and
+multi\-threading it would be slower than not doing it at all.
+Multi\-threading trades a little compression ratio for speed (measured at \fBzstd,3\fP:
++0.05% archive size for 1MiB chunks, +0.64% for 8MiB ones, more at higher levels), and
+it uses more cpu time in total to reduce the wallclock time. Set it to 1 if you would
+rather have the smaller archive, or if borg has to share the cpu with other work.
+.TP
+.B BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL
+Select the scan kernel the \fBfastcdc\fP / \fBbuzhash64\fP chunker uses (default:
+\fBscalar\fP, the plain sequential loop). Accepted values are \fBavx512\fP, \fBavx2\fP,
+\fBneon\fP, \fBblockwise\fP and \fBscalar\fP\&.
+All kernels chunk identically \- same cut points, same chunk ids \- and differ only in
+speed, so this is safe to change at any time, also for an existing repository.
+Which kernel is fastest is not predictable from the instruction set: it depends on the
+cpu and on the compiler that built borg, and the sequential loop wins on some machines.
+Nothing is selected automatically, so measure on your own hardware with
+\fBborg benchmark cpu \-\-chunking\fP before setting these.
+\fBavx512\fP and \fBavx2\fP exist only on x86\-64, \fBneon\fP only on aarch64, and only if the
+compiler that built borg supported them; \fBscalar\fP and \fBblockwise\fP are portable C
+and always available.
+Requesting a kernel that this build or this cpu cannot run is an error rather than a
+silent fallback, so a benchmark can not accidentally measure a different kernel.
+\fBborg create \-\-debug\fP logs the chunker and the kernel it was created with.
+.TP
+.B BORG_AES_CHUNKER_KERNEL
+Select the scan kernel used by the AES based chunkers \- one variable for all three of
+\fBtoeplitz\-aes\fP, \fBrabin\-aes\fP and \fBgoldilocks\-aes\fP (default: \fBevp\fP, the portable
+OpenSSL path). Accepted values are \fBvaes\fP, \fBaes\-ni\fP, \fBaes\-arm64\fP and \fBevp\fP\&.
+As with the chunker kernels above, all of them chunk identically and differ only in
+speed, nothing is selected automatically, and a kernel that can not run here is an
+error rather than a silent fallback.
+\fBvaes\fP and \fBaes\-ni\fP exist only on x86\-64, \fBaes\-arm64\fP only on aarch64.
+\fBvaes\fP additionally needs a compiler that knows it (gcc >= 11 / clang >= 14), so a
+cpu supporting VAES is not by itself enough to have that kernel available.
+.TP
+.B BORG_SHOW_SYSINFO
+When set to no (default: yes), system information (like OS, Python version, ...) in
+exceptions is not shown.
+Please only use for good reasons as it makes issues harder to analyze.
+.TP
+.B BORG_MSGPACK_VERSION_CHECK
+Controls whether Borg checks the \fBmsgpack\fP version.
+The default is \fByes\fP (strict check). Set to \fBno\fP to disable the version check and
+allow any installed \fBmsgpack\fP version. Use this at your own risk; malfunctioning or
+incompatible \fBmsgpack\fP versions may cause subtle bugs or repository data corruption.
+.TP
+.B BORG_FUSE_IMPL
+Choose the low\-level FUSE implementation borg shall use for \fBborg mount\fP\&.
+This is a comma\-separated list of implementation names, they are tried in the
+given order, e.g.:
+.INDENT 7.0
+.IP \(bu 2
+\fBmfusepy,pyfuse3,llfuse\fP: default, first try to load mfusepy, then pyfuse3, then llfuse.
+.IP \(bu 2
+\fBllfuse,pyfuse3\fP: first try to load llfuse, then try to load pyfuse3.
+.IP \(bu 2
+\fBmfusepy\fP: only try to load mfusepy
+.IP \(bu 2
+\fBpyfuse3\fP: only try to load pyfuse3
+.IP \(bu 2
+\fBllfuse\fP: only try to load llfuse
+.IP \(bu 2
+\fBnone\fP: do not try to load an implementation
+.UNINDENT
+.TP
+.B BORG_SELFTEST
+This can be used to influence borg\(aqs built\-in self\-tests. The default is to execute the tests
+at the beginning of each borg command invocation.
+.sp
+BORG_SELFTEST=disabled can be used to switch off the tests and rather save some time.
+Disabling is not recommended for normal borg users, but large scale borg storage providers can
+use this to optimize production servers after at least doing a one\-time test borg (with
+self\-tests not disabled) when installing or upgrading machines/OS/Borg.
+.TP
+.B BORG_WORKAROUNDS
+A list of comma\-separated strings that trigger workarounds in borg,
+e.g. to work around bugs in other software.
+.sp
+Currently known strings are:
+.INDENT 7.0
+.TP
+.B basesyncfile
+Use the more simple BaseSyncFile code to avoid issues with sync_file_range.
+You might need this to run borg on WSL (Windows Subsystem for Linux) or
+in systemd.nspawn containers on some architectures (e.g. ARM).
+Using this does not affect data safety, but might result in a more bursty
+write\-to\-disk behavior (not continuously streaming to disk).
+.TP
+.B retry_erofs
+Retry opening a file without O_NOATIME if opening a file with O_NOATIME
+caused EROFS. You will need this to make archives from volume shadow copies
+in WSL1 (Windows Subsystem for Linux 1).
+.TP
+.B authenticated_no_key
+Work around a lost passphrase or key for an \fBauthenticated\-*\fP mode repository
+(these are only authenticated, but not encrypted).
+If the key is missing in the repository config, add \fBkey = anything\fP there.
+.sp
+Without the key, borg can not verify anything that needs it: neither the
+authentication tag of the repository objects nor the chunk ids. It therefore
+reads the repository \fBunverified\fP \- a corrupted or tampered repository will
+not be detected. (This only concerns the \fBauthenticated\-*\fP modes; the
+\fBnone\-*\fP modes need no key and keep verifying their checksums.)
+.sp
+This workaround is \fBonly\fP for emergencies and \fBonly\fP to extract data
+from an affected repository (read\-only access):
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.EX
+BORG_WORKAROUNDS=authenticated_no_key borg extract \-\-repo repo archive
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+After you have extracted all data you need, you MUST delete the repository:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.EX
+BORG_WORKAROUNDS=authenticated_no_key borg delete repo
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+Now you can init a fresh repo. Make sure you do not use the workaround any more.
+.UNINDENT
+.UNINDENT
+.TP
+.B Output formatting:
+.INDENT 7.0
+.TP
+.B BORG_CHECK_FORMAT
+Giving the default value for \fBborg check \-\-format=X\fP\&.
+.TP
+.B BORG_LIST_FORMAT
+Giving the default value for \fBborg list \-\-format=X\fP\&.
+.TP
+.B BORG_REPO_LIST_FORMAT
+Giving the default value for \fBborg repo\-list \-\-format=X\fP\&.
+.TP
+.B BORG_PRUNE_FORMAT
+Giving the default value for \fBborg prune \-\-format=X\fP\&.
+.UNINDENT
+.TP
+.B Some automatic \(dqanswerers\(dq (if set, they automatically answer confirmation questions):
+.INDENT 7.0
+.TP
+.B BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes)
+For \(dqWarning: Attempting to access a previously unknown unencrypted repository\(dq
+.TP
+.B BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes)
+For \(dqWarning: The repository at location ... was previously located at ...\(dq
+.TP
+.B BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
+For \(dqThis is a potentially dangerous function...\(dq (check \-\-repair)
+.TP
+.B BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
+For \(dqYou requested to DELETE the repository completely \fIincluding\fP all archives it contains:\(dq
+.UNINDENT
+.sp
+Note: answers are case sensitive. setting an invalid answer value might either give the default
+answer or ask you interactively, depending on whether retries are allowed (they by default are
+allowed). So please test your scripts interactively before making them a non\-interactive script.
+.TP
+.B Directories and files:
+Borg 2 uses the platformdirs library (\%<https://\:pypi\:.org/\:project/\:platformdirs/>) to determine
+default directory locations. This means that default paths are \fBplatform\-specific\fP:
+.INDENT 7.0
+.IP \(bu 2
+Linux: XDG Base Directory Specification paths are used (e.g. \fB~/.config/borg\fP,
+\fB~/.cache/borg\fP, \fB~/.local/share/borg\fP). \fBXDG_*\fP environment variables are
+honoured (see \%<https://\:specifications\:.freedesktop\:.org/\:basedir-spec/\:0\:.6/\:ar01s03\:.html>).
+.IP \(bu 2
+macOS: native macOS directories are used by default (e.g. \fB~/Library/Application Support/borg\fP,
+\fB~/Library/Caches/borg\fP). \fBXDG_*\fP environment variables are honoured if set.
+.IP \(bu 2
+Windows: Windows AppData directories are used (e.g. \fBC:\eUsers\e<user>\eAppData\eRoaming\eborg\fP,
+\fBC:\eUsers\e<user>\eAppData\eLocal\eborg\fP). \fBXDG_*\fP environment variables are \fBnot\fP honoured.
+.UNINDENT
+.sp
+On all platforms, you can override each directory individually using the specific environment
+variables described below. You can also set \fBBORG_BASE_DIR\fP to force borg to use
+\fBBORG_BASE_DIR/.config/borg\fP, \fBBORG_BASE_DIR/.cache/borg\fP, etc., regardless of the platform.
+.sp
+Default directory locations by platform (when no \fBBORG_*\fP environment variables are set):
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.EX
+Directory  Linux                 macOS                                 Windows
+Config     ~/.config/borg        ~/Library/Application Support/borg    %APPDATA%\eborg
+Cache      ~/.cache/borg         ~/Library/Caches/borg                 %LOCALAPPDATA%\eborg\eCache
+Data       ~/.local/share/borg   ~/Library/Application Support/borg    %LOCALAPPDATA%\eborg
+Runtime    /run/user/<uid>/borg  ~/Library/Caches/TemporaryItems/borg  %TEMP%\eborg
+Keys       <config_dir>/keys     <config_dir>/keys                     <config_dir>\ekeys
+Security   <data_dir>/security   <data_dir>/security                   <data_dir>\esecurity
+.EE
+.UNINDENT
+.UNINDENT
+.INDENT 7.0
+.TP
+.B BORG_BASE_DIR
+Defaults to \fB$HOME\fP or \fB~$USER\fP or \fB~\fP (in that order).
+If you want to move all borg\-specific folders to a custom path at once, all you need to do is
+to modify \fBBORG_BASE_DIR\fP: the other paths for cache, config etc. will adapt accordingly
+(assuming you didn\(aqt set them to a different custom value).
+.TP
+.B BORG_CACHE_DIR
+Defaults to the platform\-specific cache directory (see table above).
+If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.cache/borg\fP\&.
+On Linux and macOS, \fBXDG_CACHE_HOME\fP is also honoured if \fBBORG_BASE_DIR\fP is not set.
+This directory contains the local cache and might need a lot
+of space for dealing with big repositories. Make sure you\(aqre aware of the associated
+security aspects of the cache location: \fIcache_security\fP
+.TP
+.B BORG_CONFIG_DIR
+Defaults to the platform\-specific config directory (see table above).
+If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.config/borg\fP\&.
+On Linux and macOS, \fBXDG_CONFIG_HOME\fP is also honoured if \fBBORG_BASE_DIR\fP is not set.
+This directory contains all borg configuration directories, see the FAQ
+for a security advisory about the data in this directory: \fIhome_config_borg\fP
+.TP
+.B BORG_DATA_DIR
+Defaults to the platform\-specific data directory (see table above).
+If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.local/share/borg\fP\&.
+On Linux and macOS, \fBXDG_DATA_HOME\fP is also honoured if \fBBORG_BASE_DIR\fP is not set.
+This directory contains all borg data directories, see the FAQ
+for a security advisory about the data in this directory: \fIhome_data_borg\fP
+.TP
+.B BORG_RUNTIME_DIR
+Defaults to the platform\-specific runtime directory (see table above).
+If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.cache/borg\fP\&.
+On Linux and macOS, \fBXDG_RUNTIME_DIR\fP is also honoured if \fBBORG_BASE_DIR\fP is not set.
+This directory contains borg runtime files, like e.g. the socket file.
+.TP
+.B BORG_SECURITY_DIR
+Defaults to \fB$BORG_DATA_DIR/security\fP\&.
+This directory contains security relevant data.
+.TP
+.B BORG_KEYS_DIR
+Defaults to \fB$BORG_CONFIG_DIR/keys\fP\&.
+This directory contains keys for encrypted repositories.
+.TP
+.B BORG_KEY_FILE
+When set, use the given path as repository key file. Please note that this is only
+for rather special applications that externally fully manage the key files:
+.INDENT 7.0
+.IP \(bu 2
+this setting only applies to the keyfile modes (not to the repokey modes).
+.IP \(bu 2
+using a full, absolute path to the key file is recommended.
+.IP \(bu 2
+all directories in the given path must exist.
+.IP \(bu 2
+this setting forces borg to use the key file at the given location.
+.IP \(bu 2
+the key file must either exist (for most commands) or will be created (\fBborg repo\-create\fP).
+.IP \(bu 2
+you need to give a different path for different repositories.
+.IP \(bu 2
+you need to point to the correct key file matching the repository the command will operate on.
+.UNINDENT
+.TP
+.B TMPDIR
+This is where temporary files are stored (might need a lot of temporary space for some
+operations), see \%<https://\:docs\:.python\:.org/\:3/\:library/\:tempfile\:.html#\:tempfile\:.gettempdir>
+for details.
+.UNINDENT
+.TP
+.B Building:
+.INDENT 7.0
+.TP
+.B BORG_OPENSSL_NAME
+Defines the subdirectory name for OpenSSL (setup.py).
+.TP
+.B BORG_OPENSSL_PREFIX
+Adds given OpenSSL header file directory to the default locations (setup.py).
+.TP
+.B BORG_LIBACL_PREFIX
+Adds given prefix directory to the default locations. If an \(aqinclude/acl/libacl.h\(aq is found
+Borg will be linked against the system libacl instead of a bundled implementation. (setup.py)
+.TP
+.B BORG_LIBLZ4_PREFIX
+Adds given prefix directory to the default locations. If a \(aqinclude/lz4.h\(aq is found Borg
+will be linked against the system liblz4 instead of a bundled implementation. (setup.py)
+.UNINDENT
+.TP
+.B Automatic option environment variables:
+Borg uses jsonargparse (\%<https://\:jsonargparse\:.readthedocs\:.io/>) with \fBdefault_env=True\fP,
+which means that every command\-line option can also be set via an environment variable.
+.sp
+The environment variable name is derived from the program name (\fBborg\fP),
+the subcommand (if any), and the option name, all converted to uppercase
+with dashes replaced by underscores.
+.sp
+For \fBtop\-level options\fP (not specific to a subcommand), the pattern is:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.EX
+BORG_<OPTION>
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+For example, \fB\-\-lock\-wait\fP can be set via \fBBORG_LOCK_WAIT\fP\&.
+.sp
+For \fBsubcommand options\fP, the subcommand and option are separated by a
+double underscore:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.EX
+BORG_<SUBCOMMAND>__<OPTION>
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+For example, \fBborg create \-\-comment\fP can be set via \fBBORG_CREATE__COMMENT\fP\&.
+.UNINDENT
+.sp
+Please note:
+.INDENT 0.0
+.IP \(bu 2
+Be very careful when using the \(dqyes\(dq sayers, the warnings with prompt exist for your / your data\(aqs security/safety.
+.IP \(bu 2
+Also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions (e.g.
+mode 600, root:root).
+.UNINDENT
+.SH Author
+The Borg Collective
+.\" End of generated man page.

From cfd368624e2948d5f73947173ebd959fbf3f8117 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 13:44:36 +0200
Subject: [PATCH 120/144] find: new command to search files across archives,
 #9974

borg find iterates over the archives selected by the usual archive
filter options (all archives by default) from newest to oldest, over
all items of each archive, and outputs one line per item matching the
given PATHs (paths or patterns, matching like in borg list /
borg extract).

Output lines are formatted like borg list default output, but are
prefixed with a short archive ID and the archive name. As the
archives of a series all share the same name, only the archive ID
uniquely identifies the archive.

Output is customizable via --format / BORG_FIND_FORMAT (ItemFormatter
keys) and --json-lines.

There is no extra index for file paths (keeping one up to date would
cost considerable runtime and space), so this reads the metadata of
all selected archives.
---
 docs/man/borg-find.1                         | 259 +++++++++++++++++++
 docs/usage.rst                               |   1 +
 docs/usage/find.rst                          |  21 ++
 docs/usage/find.rst.inc                      | 194 ++++++++++++++
 docs/usage/general/environment.rst.inc       |   2 +
 src/borg/archiver/__init__.py                |   3 +
 src/borg/archiver/find_cmd.py                | 134 ++++++++++
 src/borg/archiver/help_cmd.py                |   2 +
 src/borg/manifest.py                         |   5 +-
 src/borg/testsuite/archiver/find_cmd_test.py | 132 ++++++++++
 10 files changed, 751 insertions(+), 2 deletions(-)
 create mode 100644 docs/man/borg-find.1
 create mode 100644 docs/usage/find.rst
 create mode 100644 docs/usage/find.rst.inc
 create mode 100644 src/borg/archiver/find_cmd.py
 create mode 100644 src/borg/testsuite/archiver/find_cmd_test.py

diff --git a/docs/man/borg-find.1 b/docs/man/borg-find.1
new file mode 100644
index 0000000000..2d0135dd14
--- /dev/null
+++ b/docs/man/borg-find.1
@@ -0,0 +1,259 @@
+.\" Man page generated from reStructuredText
+.\" by the Docutils 0.22.4 manpage writer.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "borg-find" "1" "2026-08-14" "" "borg backup tool"
+.SH Name
+borg-find \- Find files across archives.
+.SH SYNOPSIS
+.sp
+borg [common options] find [options] [PATH...]
+.SH DESCRIPTION
+.sp
+This command finds files matching the given paths or patterns in the archives
+selected by the usual archive filter options (all archives, if no filters are
+given). It iterates over the matching archives from newest to oldest, over all
+items of each archive, and outputs one line per match, like \fBborg list\fP, but
+prefixed with a short archive ID and the archive name. As the archives of a
+series all share the same name, only the archive ID uniquely identifies the
+archive.
+.sp
+This makes it easy to answer questions like \(dqwhich archives contain this file?\(dq
+or \(dqwhere did that file end up?\(dq:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.EX
+$ borg find home/user/file.txt          # in which archives is this file?
+$ borg find \(aqsh:**/*.jpg\(aq \-\-last 3      # all jpg files in the last 3 archives
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+The given PATHs match like in \fBborg list\fP or \fBborg extract\fP: a plain path
+matches the item with that path as well as everything below it, and the pattern
+styles (\fBfm:\fP, \fBsh:\fP, \fBre:\fP, \fBpp:\fP, \fBpf:\fP) are supported as well.
+For more help on include/exclude patterns, see the output of \fIborg_patterns\fP\&.
+.sp
+Note: there is no extra index for the file paths, so this command reads the
+metadata of all selected archives, which may take a while for many/big archives.
+.SH OPTIONS
+.sp
+See \fIborg\-common(1)\fP for common options of Borg commands.
+.SS arguments
+.INDENT 0.0
+.TP
+.B PATH
+paths to find; patterns are supported
+.UNINDENT
+.SS options
+.INDENT 0.0
+.TP
+.BI \-\-format \ FORMAT
+specify format for file listing (default: \(dq{archiveid:.8} {archivename} {mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}\(dq)
+.TP
+.B  \-\-json\-lines
+Format output as JSON Lines. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text.
+.UNINDENT
+.SS Archive filters
+.INDENT 0.0
+.TP
+.BI \-a \ PATTERN\fR,\fB \ \-\-match\-archives \ PATTERN
+only consider archives matching all patterns. See \(dqborg help match\-archives\(dq.
+.TP
+.BI \-\-sort\-by \ KEYS
+Comma\-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp
+.TP
+.BI \-\-first \ N
+consider the first N archives after other filters are applied
+.TP
+.BI \-\-last \ N
+consider the last N archives after other filters are applied
+.TP
+.BI \-\-oldest \ TIMESPAN
+consider archives between the oldest archive\(aqs timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.
+.TP
+.BI \-\-newest \ TIMESPAN
+consider archives between the newest archive\(aqs timestamp and (newest \- TIMESPAN), e.g., 7d or 12m.
+.TP
+.BI \-\-older \ TIMESPAN
+consider archives older than (now \- TIMESPAN), e.g., 7d or 12m.
+.TP
+.BI \-\-newer \ TIMESPAN
+consider archives newer than (now \- TIMESPAN), e.g., 7d or 12m.
+.UNINDENT
+.SS Include/Exclude options
+.INDENT 0.0
+.TP
+.BI \-e \ PATTERN\fR,\fB \ \-\-exclude \ PATTERN
+exclude paths matching PATTERN
+.TP
+.BI \-\-exclude\-from \ EXCLUDEFILE
+read exclude patterns from EXCLUDEFILE, one per line
+.TP
+.BI \-\-pattern \ PATTERN
+include/exclude paths matching PATTERN
+.TP
+.BI \-\-patterns\-from \ PATTERNFILE
+read include/exclude patterns from PATTERNFILE, one per line
+.UNINDENT
+.SH EXAMPLES
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.EX
+# In which archives is this file? Searched / printed from newest to oldest archive.
+$ borg find home/user/file.txt
+41a2ed21 docs \-rw\-rw\-r\-\- user   user       1522 Sun, 2022\-02\-06 21:02:18 home/user/file.txt
+20e70e3a docs \-rw\-rw\-r\-\- user   user       1440 Sun, 2022\-01\-30 20:47:32 home/user/file.txt
+
+# Find all jpg files in the last 3 archives.
+$ borg find \(aqsh:**/*.jpg\(aq \-\-last 3
+39c8956e photos \-rw\-rw\-r\-\- user   user     919337 Sat, 2022\-01\-01 14:20:21 photos/paris/eiffel.jpg
+39c8956e photos \-rw\-rw\-r\-\- user   user    1023881 Sun, 2022\-02\-06 09:12:44 photos/rome/colosseum.jpg
+04061a53 photos \-rw\-rw\-r\-\- user   user     919337 Sat, 2022\-01\-01 14:20:21 photos/paris/eiffel.jpg
+
+# Only print the archive and the path, nothing else.
+$ borg find \-\-format \(aq{archiveid:.8} {archivename} {path}{NL}\(aq home/user/file.txt
+41a2ed21 docs home/user/file.txt
+20e70e3a docs home/user/file.txt
+.EE
+.UNINDENT
+.UNINDENT
+.SH NOTES
+.SS The FORMAT specifier syntax
+.sp
+The \fB\-\-format\fP option uses Python\(aqs format string syntax \%<https://\:docs\:.python\:.org/\:3\:.11/\:library/\:string\:.html#\:formatstrings>\&.
+.sp
+Examples:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.EX
+# only print the archive and the path, nothing else
+$ borg find \-\-format \(aq{archiveid:.8} {archivename} {path}{NL}\(aq \(aqsh:**/*.jpg\(aq
+20e70e3a photos photos/paris/eiffel.jpg
+\&...
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+{archiveid:.8} prints a short archive ID (use {archiveid} for the full ID),
+{archivename} the archive name (the archives of a series all share the name).
+.sp
+The following keys are always available:
+\- NEWLINE: OS dependent line separator
+\- NL: alias of NEWLINE
+\- NUL: NUL character for creating print0 / xargs \-0 like output
+\- SPACE: space character
+\- TAB: tab character
+\- CR: carriage return character
+\- LF: line feed character
+.sp
+Keys available only when finding files in an archive:
+.INDENT 0.0
+.IP \(bu 2
+type: file type (file, dir, symlink, ...)
+.IP \(bu 2
+mode: file mode (as in stat)
+.IP \(bu 2
+uid: user id of file owner
+.IP \(bu 2
+gid: group id of file owner
+.IP \(bu 2
+user: user name of file owner
+.IP \(bu 2
+group: group name of file owner
+.IP \(bu 2
+path: file path
+.IP \(bu 2
+target: link target for symlinks
+.IP \(bu 2
+hlid: hard link identity (same if hardlinking same fs object)
+.IP \(bu 2
+inode: inode number
+.IP \(bu 2
+flags: file flags
+.IP \(bu 2
+size: file size
+.IP \(bu 2
+num_chunks: number of chunks in this file
+.IP \(bu 2
+mtime: file modification time
+.IP \(bu 2
+ctime: file change time
+.IP \(bu 2
+atime: file access time
+.IP \(bu 2
+isomtime: file modification time (ISO 8601 format)
+.IP \(bu 2
+isoctime: file change time (ISO 8601 format)
+.IP \(bu 2
+isoatime: file access time (ISO 8601 format)
+.IP \(bu 2
+fingerprint: Fingerprint of the file content (may have false negatives), format: H(conditions)\-H(chunk_ids)
+.IP \(bu 2
+blake2b
+.IP \(bu 2
+blake2s
+.IP \(bu 2
+blake3
+.IP \(bu 2
+md5
+.IP \(bu 2
+sha1
+.IP \(bu 2
+sha224
+.IP \(bu 2
+sha256
+.IP \(bu 2
+sha384
+.IP \(bu 2
+sha3_224
+.IP \(bu 2
+sha3_256
+.IP \(bu 2
+sha3_384
+.IP \(bu 2
+sha3_512
+.IP \(bu 2
+sha512
+.IP \(bu 2
+archiveid: internal ID of the archive
+.IP \(bu 2
+archivename: name of the archive
+.IP \(bu 2
+extra: prepends {target} with \(dq \-> \(dq for soft links and \(dq link to \(dq for hard links
+.UNINDENT
+.SH SEE ALSO
+.sp
+\fIborg\-common(1)\fP
+.SH Author
+The Borg Collective
+.\" End of generated man page.
diff --git a/docs/usage.rst b/docs/usage.rst
index 56c63558b8..d049fe82b3 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -52,6 +52,7 @@ Usage
    usage/extract
    usage/check
    usage/list
+   usage/find
    usage/tag
    usage/rename
    usage/diff
diff --git a/docs/usage/find.rst b/docs/usage/find.rst
new file mode 100644
index 0000000000..dc7e7041c1
--- /dev/null
+++ b/docs/usage/find.rst
@@ -0,0 +1,21 @@
+.. include:: find.rst.inc
+
+Examples
+~~~~~~~~
+::
+
+    # In which archives is this file? Searched / printed from newest to oldest archive.
+    $ borg find home/user/file.txt
+    41a2ed21 docs -rw-rw-r-- user   user       1522 Sun, 2022-02-06 21:02:18 home/user/file.txt
+    20e70e3a docs -rw-rw-r-- user   user       1440 Sun, 2022-01-30 20:47:32 home/user/file.txt
+
+    # Find all jpg files in the last 3 archives.
+    $ borg find 'sh:**/*.jpg' --last 3
+    39c8956e photos -rw-rw-r-- user   user     919337 Sat, 2022-01-01 14:20:21 photos/paris/eiffel.jpg
+    39c8956e photos -rw-rw-r-- user   user    1023881 Sun, 2022-02-06 09:12:44 photos/rome/colosseum.jpg
+    04061a53 photos -rw-rw-r-- user   user     919337 Sat, 2022-01-01 14:20:21 photos/paris/eiffel.jpg
+
+    # Only print the archive and the path, nothing else.
+    $ borg find --format '{archiveid:.8} {archivename} {path}{NL}' home/user/file.txt
+    41a2ed21 docs home/user/file.txt
+    20e70e3a docs home/user/file.txt
diff --git a/docs/usage/find.rst.inc b/docs/usage/find.rst.inc
new file mode 100644
index 0000000000..b086ebb7c3
--- /dev/null
+++ b/docs/usage/find.rst.inc
@@ -0,0 +1,194 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_find:
+
+borg find
+---------
+.. code-block:: none
+
+    borg [common options] find [options] [PATH...]
+
+.. only:: html
+
+    .. class:: borg-options-table
+
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **positional arguments**                                                                                                                                                                                                                                                                                           |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``PATH``                                     | paths to find; patterns are supported                                                                                                                                                 |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **options**                                                                                                                                                                                                                                                                                                        |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--format FORMAT``                          | specify format for file listing (default: "{archiveid:.8} {archivename} {mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}")                                                |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--json-lines``                             | Format output as JSON Lines. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | .. class:: borg-common-opt-ref                                                                                                                                                                                                                                                                                     |
+    |                                                                                                                                                                                                                                                                                                                    |
+    | :ref:`common_options`                                                                                                                                                                                                                                                                                              |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **Archive filters** — Archive filters can be applied to repository targets.                                                                                                                                                                                                                                        |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``-a PATTERN``, ``--match-archives PATTERN`` | only consider archives matching all patterns. See "borg help match-archives".                                                                                                         |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--sort-by KEYS``                           | Comma-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp                                                           |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--first N``                                | consider the first N archives after other filters are applied                                                                                                                         |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--last N``                                 | consider the last N archives after other filters are applied                                                                                                                          |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--oldest TIMESPAN``                        | consider archives between the oldest archive's timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.                                                                                    |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--newest TIMESPAN``                        | consider archives between the newest archive's timestamp and (newest - TIMESPAN), e.g., 7d or 12m.                                                                                    |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--older TIMESPAN``                         | consider archives older than (now - TIMESPAN), e.g., 7d or 12m.                                                                                                                       |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--newer TIMESPAN``                         | consider archives newer than (now - TIMESPAN), e.g., 7d or 12m.                                                                                                                       |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    | **Include/Exclude options**                                                                                                                                                                                                                                                                                        |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``-e PATTERN``, ``--exclude PATTERN``        | exclude paths matching PATTERN                                                                                                                                                        |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--exclude-from EXCLUDEFILE``               | read exclude patterns from EXCLUDEFILE, one per line                                                                                                                                  |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--pattern PATTERN``                        | include/exclude paths matching PATTERN                                                                                                                                                |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                                             | ``--patterns-from PATTERNFILE``              | read include/exclude patterns from PATTERNFILE, one per line                                                                                                                          |
+    +-----------------------------------------------------------------------------+----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+
+    .. raw:: html
+
+        <script type='text/javascript'>
+        $(document).ready(function () {
+            $('.borg-options-table colgroup').remove();
+        })
+        </script>
+
+.. only:: latex
+
+    PATH
+        paths to find; patterns are supported
+
+
+    options
+        --format FORMAT    specify format for file listing (default: "{archiveid:.8} {archivename} {mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}")
+        --json-lines     Format output as JSON Lines. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text.
+
+
+    :ref:`common_options`
+        |
+
+    Archive filters
+        -a PATTERN, --match-archives PATTERN     only consider archives matching all patterns. See "borg help match-archives".
+        --sort-by KEYS                           Comma-separated list of sorting keys; valid keys are: timestamp, archive, name, id, tags, host, user; default is: timestamp
+        --first N                                consider the first N archives after other filters are applied
+        --last N                                 consider the last N archives after other filters are applied
+        --oldest TIMESPAN                        consider archives between the oldest archive's timestamp and (oldest + TIMESPAN), e.g., 7d or 12m.
+        --newest TIMESPAN                        consider archives between the newest archive's timestamp and (newest - TIMESPAN), e.g., 7d or 12m.
+        --older TIMESPAN                         consider archives older than (now - TIMESPAN), e.g., 7d or 12m.
+        --newer TIMESPAN                         consider archives newer than (now - TIMESPAN), e.g., 7d or 12m.
+
+
+    Include/Exclude options
+        -e PATTERN, --exclude PATTERN     exclude paths matching PATTERN
+        --exclude-from EXCLUDEFILE        read exclude patterns from EXCLUDEFILE, one per line
+        --pattern PATTERN                 include/exclude paths matching PATTERN
+        --patterns-from PATTERNFILE       read include/exclude patterns from PATTERNFILE, one per line
+
+
+Description
+~~~~~~~~~~~
+
+This command finds files matching the given paths or patterns in the archives
+selected by the usual archive filter options (all archives, if no filters are
+given). It iterates over the matching archives from newest to oldest, over all
+items of each archive, and outputs one line per match, like ``borg list``, but
+prefixed with a short archive ID and the archive name. As the archives of a
+series all share the same name, only the archive ID uniquely identifies the
+archive.
+
+This makes it easy to answer questions like "which archives contain this file?"
+or "where did that file end up?"::
+
+    $ borg find home/user/file.txt          # in which archives is this file?
+    $ borg find 'sh:**/*.jpg' --last 3      # all jpg files in the last 3 archives
+
+The given PATHs match like in ``borg list`` or ``borg extract``: a plain path
+matches the item with that path as well as everything below it, and the pattern
+styles (``fm:``, ``sh:``, ``re:``, ``pp:``, ``pf:``) are supported as well.
+For more help on include/exclude patterns, see the output of :ref:`borg_patterns`.
+
+Note: there is no extra index for the file paths, so this command reads the
+metadata of all selected archives, which may take a while for many/big archives.
+
+.. man NOTES
+
+The FORMAT specifier syntax
++++++++++++++++++++++++++++
+
+The ``--format`` option uses Python's `format string syntax
+<https://docs.python.org/3.11/library/string.html#formatstrings>`_.
+
+Examples:
+::
+
+    # only print the archive and the path, nothing else
+    $ borg find --format '{archiveid:.8} {archivename} {path}{NL}' 'sh:**/*.jpg'
+    20e70e3a photos photos/paris/eiffel.jpg
+    ...
+
+{archiveid:.8} prints a short archive ID (use {archiveid} for the full ID),
+{archivename} the archive name (the archives of a series all share the name).
+
+The following keys are always available:
+- NEWLINE: OS dependent line separator
+- NL: alias of NEWLINE
+- NUL: NUL character for creating print0 / xargs -0 like output
+- SPACE: space character
+- TAB: tab character
+- CR: carriage return character
+- LF: line feed character
+
+
+Keys available only when finding files in an archive:
+
+- type: file type (file, dir, symlink, ...)
+- mode: file mode (as in stat)
+- uid: user id of file owner
+- gid: group id of file owner
+- user: user name of file owner
+- group: group name of file owner
+- path: file path
+- target: link target for symlinks
+- hlid: hard link identity (same if hardlinking same fs object)
+- inode: inode number
+- flags: file flags
+
+- size: file size
+- num_chunks: number of chunks in this file
+
+- mtime: file modification time
+- ctime: file change time
+- atime: file access time
+- isomtime: file modification time (ISO 8601 format)
+- isoctime: file change time (ISO 8601 format)
+- isoatime: file access time (ISO 8601 format)
+
+- fingerprint: Fingerprint of the file content (may have false negatives), format: H(conditions)-H(chunk_ids)
+- blake2b
+- blake2s
+- blake3
+- md5
+- sha1
+- sha224
+- sha256
+- sha384
+- sha3_224
+- sha3_256
+- sha3_384
+- sha3_512
+- sha512
+
+- archiveid: internal ID of the archive
+- archivename: name of the archive
+- extra: prepends {target} with " -> " for soft links and " link to " for hard links
diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc
index d9ac6db6db..1f4fcdcc97 100644
--- a/docs/usage/general/environment.rst.inc
+++ b/docs/usage/general/environment.rst.inc
@@ -288,6 +288,8 @@ General:
 Output formatting:
     BORG_CHECK_FORMAT
         Giving the default value for ``borg check --format=X``.
+    BORG_FIND_FORMAT
+        Giving the default value for ``borg find --format=X``.
     BORG_LIST_FORMAT
         Giving the default value for ``borg list --format=X``.
     BORG_REPO_LIST_FORMAT
diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py
index e3a35c99be..c8730977eb 100644
--- a/src/borg/archiver/__init__.py
+++ b/src/borg/archiver/__init__.py
@@ -74,6 +74,7 @@
 from .delete_cmd import DeleteMixIn
 from .diff_cmd import DiffMixIn
 from .extract_cmd import ExtractMixIn
+from .find_cmd import FindMixIn
 from .help_cmd import HelpMixIn
 from .info_cmd import InfoMixIn
 from .key_cmds import KeysMixIn
@@ -109,6 +110,7 @@ class Archiver(
     DeleteMixIn,
     DiffMixIn,
     ExtractMixIn,
+    FindMixIn,
     HelpMixIn,
     InfoMixIn,
     KeysMixIn,
@@ -301,6 +303,7 @@ def build_parser(self):
         self.build_parser_delete(subparsers, common_parser, mid_common_parser)
         self.build_parser_diff(subparsers, common_parser, mid_common_parser)
         self.build_parser_extract(subparsers, common_parser, mid_common_parser)
+        self.build_parser_find(subparsers, common_parser, mid_common_parser)
         self.build_parser_help(subparsers, common_parser, mid_common_parser, parser)
         self.build_parser_info(subparsers, common_parser, mid_common_parser)
         self.build_parser_keys(subparsers, common_parser, mid_common_parser)
diff --git a/src/borg/archiver/find_cmd.py b/src/borg/archiver/find_cmd.py
new file mode 100644
index 0000000000..7432c7b284
--- /dev/null
+++ b/src/borg/archiver/find_cmd.py
@@ -0,0 +1,134 @@
+import os
+import textwrap
+import sys
+
+from ._common import with_repository, build_matcher, define_archive_filters_group, Highlander
+from ..archive import Archive
+from ..cache import Cache
+from ..constants import *  # NOQA
+from ..helpers import ItemFormatter, BaseFormatter, PathSpec
+from ..helpers.argparsing import ArgumentParser
+from ..manifest import Manifest
+
+from ..logger import create_logger
+
+logger = create_logger()
+
+
+class FindMixIn:
+    @with_repository(compatibility=(Manifest.Operation.READ,))
+    def do_find(self, args, repository, manifest):
+        """Find files across archives."""
+        matcher = build_matcher(args.patterns, args.paths)
+        if args.format is not None:
+            format = args.format
+        else:
+            format = os.environ.get(
+                "BORG_FIND_FORMAT",
+                "{archiveid:.8} {archivename} {mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}",
+            )
+        # check the format before doing any work with it (also: the ItemFormatter is only built later)
+        ItemFormatter.validate_format(format)
+
+        archive_infos = manifest.archives.list_considering(args, reverse=True)
+        num_archives = len(archive_infos)
+
+        def _find_inner(cache):
+            for i, info in enumerate(archive_infos):
+                logger.info(f"Searching archive {info.name} {info.ts.astimezone()} ({i + 1}/{num_archives})")
+                archive = Archive(manifest, info.id, cache=cache)
+                formatter = ItemFormatter(archive, format)
+                for item in archive.iter_items(lambda item: matcher.match(item.path)):
+                    sys.stdout.write(formatter.format_item(item, args.json_lines, sort=True))
+
+        # Only load the cache if it will be used
+        if ItemFormatter.format_needs_cache(format):
+            with Cache(repository, manifest) as cache:
+                _find_inner(cache)
+        else:
+            _find_inner(cache=None)
+
+    def build_parser_find(self, subparsers, common_parser, mid_common_parser):
+        from ._common import process_epilog, define_exclusion_group
+
+        find_epilog = (
+            process_epilog(
+                """
+        This command finds files matching the given paths or patterns in the archives
+        selected by the usual archive filter options (all archives, if no filters are
+        given). It iterates over the matching archives from newest to oldest, over all
+        items of each archive, and outputs one line per match, like ``borg list``, but
+        prefixed with a short archive ID and the archive name. As the archives of a
+        series all share the same name, only the archive ID uniquely identifies the
+        archive.
+
+        This makes it easy to answer questions like "which archives contain this file?"
+        or "where did that file end up?"::
+
+            $ borg find home/user/file.txt          # in which archives is this file?
+            $ borg find 'sh:**/*.jpg' --last 3      # all jpg files in the last 3 archives
+
+        The given PATHs match like in ``borg list`` or ``borg extract``: a plain path
+        matches the item with that path as well as everything below it, and the pattern
+        styles (``fm:``, ``sh:``, ``re:``, ``pp:``, ``pf:``) are supported as well.
+        For more help on include/exclude patterns, see the output of :ref:`borg_patterns`.
+
+        Note: there is no extra index for the file paths, so this command reads the
+        metadata of all selected archives, which may take a while for many/big archives.
+
+        .. man NOTES
+
+        The FORMAT specifier syntax
+        +++++++++++++++++++++++++++
+
+        The ``--format`` option uses Python's `format string syntax
+        <https://docs.python.org/3.11/library/string.html#formatstrings>`_.
+
+        Examples:
+        ::
+
+            # only print the archive and the path, nothing else
+            $ borg find --format '{archiveid:.8} {archivename} {path}{NL}' 'sh:**/*.jpg'
+            20e70e3a photos photos/paris/eiffel.jpg
+            ...
+
+        {archiveid:.8} prints a short archive ID (use {archiveid} for the full ID),
+        {archivename} the archive name (the archives of a series all share the name).
+
+        The following keys are always available:
+
+        """
+            )
+            + BaseFormatter.keys_help()
+            + textwrap.dedent(
+                """
+
+        Keys available only when finding files in an archive:
+
+        """
+            )
+            + ItemFormatter.keys_help()
+        )
+        subparser = ArgumentParser(parents=[common_parser], description=self.do_find.__doc__, epilog=find_epilog)
+        subparsers.add_subcommand("find", subparser, help="find files across archives")
+        subparser.add_argument(
+            "--format",
+            metavar="FORMAT",
+            dest="format",
+            action=Highlander,
+            help="specify format for file listing (default: "
+            '"{archiveid:.8} {archivename} {mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}")',
+        )
+        subparser.add_argument(
+            "--json-lines",
+            action="store_true",
+            help="Format output as JSON Lines. "
+            "The form of ``--format`` is ignored, "
+            "but keys used in it are added to the JSON output. "
+            "Some keys are always present. Note: JSON can only represent text.",
+        )
+        subparser.add_argument(
+            "paths", metavar="PATH", nargs="*", type=PathSpec, help="paths to find; patterns are supported"
+        )
+        define_archive_filters_group(subparser)
+        define_exclusion_group(subparser)
diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py
index 1b1ac4d564..ce34b6c06a 100644
--- a/src/borg/archiver/help_cmd.py
+++ b/src/borg/archiver/help_cmd.py
@@ -870,6 +870,8 @@ class HelpMixIn:
         Output formatting:
             BORG_CHECK_FORMAT
                 Giving the default value for ``borg check --format=X``.
+            BORG_FIND_FORMAT
+                Giving the default value for ``borg find --format=X``.
             BORG_LIST_FORMAT
                 Giving the default value for ``borg list --format=X``.
             BORG_REPO_LIST_FORMAT
diff --git a/src/borg/manifest.py b/src/borg/manifest.py
index 4881cf4780..d25070d8f4 100644
--- a/src/borg/manifest.py
+++ b/src/borg/manifest.py
@@ -120,7 +120,7 @@ def list(
         newest=None,
         deleted=False,
     ): ...
-    def list_considering(self, args): ...
+    def list_considering(self, args, *, reverse=False): ...
     def get_one(self, match, *, match_end=r"\Z", deleted=False): ...
 
 
@@ -413,7 +413,7 @@ def list(
             archive_infos.reverse()
         return archive_infos
 
-    def list_considering(self, args):
+    def list_considering(self, args, *, reverse=False):
         """
         get a list of archives, considering --first/last/prefix/match-archives/sort cmdline args
         """
@@ -424,6 +424,7 @@ def list_considering(self, args):
             )
         return self.list(
             sort_by=args.sort_by.split(","),
+            reverse=reverse,
             match=args.match_archives,
             first=getattr(args, "first", None),
             last=getattr(args, "last", None),
diff --git a/src/borg/testsuite/archiver/find_cmd_test.py b/src/borg/testsuite/archiver/find_cmd_test.py
new file mode 100644
index 0000000000..b6ff983706
--- /dev/null
+++ b/src/borg/testsuite/archiver/find_cmd_test.py
@@ -0,0 +1,132 @@
+import json
+
+from ...constants import *  # NOQA
+from . import cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION
+
+pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary")  # NOQA
+
+# terse format for tests that only care about which archives/items matched
+FMT = "{archiveid:.8} {archivename} {path}{NL}"
+
+
+def short_ids(archiver):
+    """map archive name -> short (8 hex digits) archive id, as printed by the default find output"""
+    output = cmd(archiver, "repo-list", "--json")
+    return {archive["name"]: archive["id"][:8] for archive in json.loads(output)["archives"]}
+
+
+def test_find_basic(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file1", size=1024)
+    cmd(archiver, "create", "archive1", "input")
+    create_regular_file(archiver.input_path, "file2", size=1024)
+    cmd(archiver, "create", "archive2", "input")
+    ids = short_ids(archiver)
+
+    # file1 is in both archives, output ordered from newest to oldest archive.
+    output = cmd(archiver, "find", "input/file1", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive2']} archive2 input/file1", f"{ids['archive1']} archive1 input/file1"]
+
+    # file2 is only in the second archive.
+    output = cmd(archiver, "find", "input/file2", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive2']} archive2 input/file2"]
+
+    # a path matches itself and everything below it.
+    output = cmd(archiver, "find", "input", "--format", FMT)
+    lines = output.splitlines()
+    # ordered by archive; within one archive, the items come in the order stored in the archive.
+    assert lines[0] == f"{ids['archive2']} archive2 input"
+    assert set(lines[1:3]) == {f"{ids['archive2']} archive2 input/file1", f"{ids['archive2']} archive2 input/file2"}
+    assert lines[3:] == [f"{ids['archive1']} archive1 input", f"{ids['archive1']} archive1 input/file1"]
+
+    # without any PATH, all items of all archives match.
+    assert cmd(archiver, "find", "--format", FMT) == output
+
+    # no match: no output.
+    output = cmd(archiver, "find", "input/does-not-exist", "--format", FMT)
+    assert output == ""
+
+
+def test_find_patterns(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "dir1/file.jpg", size=1)
+    create_regular_file(archiver.input_path, "dir2/file.txt", size=1)
+    cmd(archiver, "create", "archive1", "input")
+    ids = short_ids(archiver)
+
+    output = cmd(archiver, "find", "sh:**/*.jpg", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive1']} archive1 input/dir1/file.jpg"]
+
+    output = cmd(archiver, "find", "fm:*.txt", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive1']} archive1 input/dir2/file.txt"]
+
+    output = cmd(archiver, "find", "input", "--exclude", "sh:**/*.jpg", "--format", FMT)
+    assert "file.jpg" not in output
+    assert f"{ids['archive1']} archive1 input/dir2/file.txt" in output
+
+    output = cmd(archiver, "find", "--pattern", "+ re:\\.jpg$", "--pattern", "- re:^.*$", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive1']} archive1 input/dir1/file.jpg"]
+
+
+def test_find_archive_filters(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file1", size=1024)
+    cmd(archiver, "create", "archive1", "input")
+    cmd(archiver, "create", "archive2", "input")
+    ids = short_ids(archiver)
+
+    output = cmd(archiver, "find", "input/file1", "--last", "1", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive2']} archive2 input/file1"]
+
+    output = cmd(archiver, "find", "input/file1", "--first", "1", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive1']} archive1 input/file1"]
+
+    output = cmd(archiver, "find", "input/file1", "-a", "archive1", "--format", FMT)
+    assert output.splitlines() == [f"{ids['archive1']} archive1 input/file1"]
+
+
+def test_find_default_format(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file1", size=1024)
+    cmd(archiver, "create", "archive1", "input")
+    cmd(archiver, "create", "archive2", "input")
+    ids = short_ids(archiver)
+
+    # the default format is the borg list default format with "{archiveid:.8} {archivename} " prepended.
+    find_output = cmd(archiver, "find", "input/file1")
+    expected = []
+    for name in ("archive2", "archive1"):  # newest to oldest
+        for line in cmd(archiver, "list", name, "input/file1").splitlines():
+            expected.append(f"{ids[name]} {name} {line}")
+    assert find_output.splitlines() == expected
+
+
+def test_find_format(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file1", size=1024)
+    cmd(archiver, "create", "archive1", "input")
+
+    output = cmd(archiver, "find", "input/file1", "--format", "{size} {path}{NL}")
+    assert output.splitlines() == ["1024 input/file1"]
+
+
+def test_find_json_lines(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file1", size=1024)
+    cmd(archiver, "create", "archive1", "input")
+    cmd(archiver, "create", "archive2", "input")
+    ids = short_ids(archiver)
+
+    output = cmd(archiver, "find", "input/file1", "--json-lines")
+    rows = [json.loads(line) for line in output.splitlines()]
+    # the default format contains {archiveid} and {archivename}, so they are present in the JSON output.
+    assert [(row["archiveid"][:8], row["archivename"], row["path"]) for row in rows] == [
+        (ids["archive2"], "archive2", "input/file1"),
+        (ids["archive1"], "archive1", "input/file1"),
+    ]

From 448f1c6c9db600772c653db66847f7775dd30acf Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 15:55:56 +0200
Subject: [PATCH 121/144] create: support --stats with --dry-run, fixes #1648

A dry run already walks the files and stats them, so it can cheaply
count the files and sum up their sizes from file system metadata,
without reading any file contents.

--stats (and --json) with --dry-run now report the number of files
and the original size. The deduplicated size is unknown in a dry run,
as data is not read, chunked, and deduplicated. The sizes of data read
from stdin, from a command's output, and from special files are also
unknown in a dry run and counted as zero. Hardlinks count once per
link, as in a real create.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/internals/frontends.rst                  |  4 ++
 docs/usage/create.rst.inc                     | 10 +++-
 src/borg/archiver/__init__.py                 |  3 +-
 src/borg/archiver/create_cmd.py               | 53 ++++++++++++++++---
 .../testsuite/archiver/create_cmd_test.py     | 33 ++++++++++++
 5 files changed, 94 insertions(+), 9 deletions(-)

diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst
index 6af8114b2d..5cec57a0f6 100644
--- a/docs/internals/frontends.rst
+++ b/docs/internals/frontends.rst
@@ -356,6 +356,10 @@ Archive formats
 array under the *archives* key, while :ref:`borg_create` returns a single archive object under the
 *archive* key.
 
+:ref:`borg_create` with ``--dry-run`` does not create an archive, so there is no *archive* key.
+Instead, it returns *dry_run* (true) and a reduced *stats* object with *nfiles* and *original_size*,
+both computed from file system metadata without reading the file contents.
+
 Both formats contain a *name* key with the archive name, the *id* key with the hexadecimal archive ID,
 and the *start* key with the start timestamp.
 
diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc
index b914e9f204..f3a5f7b4a5 100644
--- a/docs/usage/create.rst.inc
+++ b/docs/usage/create.rst.inc
@@ -274,8 +274,14 @@ does not go to a terminal (e.g. into a logfile), the precise format is always us
 When using ``--stats``, you will get some statistics about how much data was
 added - the "This Archive" deduplicated size there is most interesting as that is
 how much your repository will grow. Please note that the "All archives" stats refer to
-the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually
-exclusive because the data is not actually compressed and deduplicated during a dry run.
+the state after creation.
+
+When ``--stats`` is used together with ``--dry-run``, only the number of files and the
+original size are reported. They are computed from file system metadata, without reading
+the file contents, so a dry run stays fast. As data is not actually read, chunked, and
+deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read
+from standard input, from a command's output, or from special files (``--read-special``)
+are also unknown in a dry run and counted as zero.
 
 The ``--stats`` output also reports the store statistics (lines prefixed with
 "Store"), taken from the storage layer after this run. These cover the backend and
diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py
index e3a35c99be..92d96d7fbd 100644
--- a/src/borg/archiver/__init__.py
+++ b/src/borg/archiver/__init__.py
@@ -473,7 +473,8 @@ def run(self, args):
         self._setup_implied_logging(vars(args))
         self._setup_topic_debugging(args)
         # extract --dry-run reads, decrypts and decompresses every object, so its store stats are accurate.
-        stats_supported_with_dry_run = func_name == "do_extract"
+        # create --dry-run counts the files and sums up their sizes from file system metadata.
+        stats_supported_with_dry_run = func_name in ("do_extract", "do_create")
         if getattr(args, "stats", False) and getattr(args, "dry_run", False) and not stats_supported_with_dry_run:
             logger.warning("Ignoring --stats. It is not supported when using --dry-run.")
             args.stats = False
diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py
index 1727ca14d2..c53c6e149d 100644
--- a/src/borg/archiver/create_cmd.py
+++ b/src/borg/archiver/create_cmd.py
@@ -10,7 +10,7 @@
 
 from ._common import with_repository, Highlander
 from .. import helpers
-from ..archive import Archive, is_special, SF_DATALESS
+from ..archive import Archive, Statistics, is_special, SF_DATALESS
 from ..archive import BackupError, BackupOSError, BackupItemExcluded, backup_io, OsOpen, stat_update_check
 from ..archive import FilesystemObjectProcessors, MetadataCollector, ChunksProcessor
 from ..cache import Cache
@@ -22,7 +22,7 @@
 from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify
 from ..helpers import dir_is_tagged
 from ..helpers import log_multi
-from ..helpers import basic_json_data, json_print
+from ..helpers import basic_json_data, json_print, FileSize
 from ..helpers import flags_dir, flags_special_follow, flags_special
 from ..helpers import prepare_subprocess_env
 from ..helpers import sig_int, ignore_sigint
@@ -97,6 +97,7 @@ def create_inner(archive, cache, fso):
                         raise Error(f"{path!r}: {e}")
                 else:
                     status = "+"  # included
+                    self.dry_run_stats.nfiles += 1  # size unknown without running the command
                 self.print_file_status(status, path)
             elif args.paths_from_command or args.paths_from_shell_command or args.paths_from_stdin:
                 paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n"
@@ -174,6 +175,7 @@ def create_inner(archive, cache, fso):
                                 status = "E"
                         else:
                             status = "+"  # included
+                            self.dry_run_stats.nfiles += 1  # size unknown without reading stdin
                         self.print_file_status(status, path)
                         if not dry_run and status is not None:
                             fso.stats.files_stats[status] += 1
@@ -226,6 +228,7 @@ def create_inner(archive, cache, fso):
         self.noxattrs = args.noxattrs
         self.exclude_dataless = args.exclude_dataless
         dry_run = args.dry_run
+        self.dry_run_stats = Statistics() if dry_run else None
         self.start_backup = time.time_ns()
         t0 = archive_ts_now()
         logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed))
@@ -288,6 +291,24 @@ def create_inner(archive, cache, fso):
                     log_multi(str(archive), str(archive.stats), logger=logging.getLogger("borg.output.stats"))
         else:
             create_inner(None, None, None)
+            args.stats |= args.json
+            if args.stats:
+                stats = self.dry_run_stats
+                if args.json:
+                    json_data = basic_json_data(
+                        manifest,
+                        extra={
+                            "dry_run": True,
+                            "stats": {"nfiles": stats.nfiles, "original_size": FileSize(stats.osize)},
+                        },
+                    )
+                    json_print(json_data)
+                else:
+                    log_multi(
+                        f"Number of files: {stats.nfiles}",
+                        f"Original size: {stats.osize_fmt}",
+                        logger=logging.getLogger("borg.output.stats"),
+                    )
 
     def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, dry_run, strip_prefix):
         """
@@ -295,6 +316,22 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
         """
 
         if dry_run:
+            stats = self.dry_run_stats
+            if stat.S_ISREG(st.st_mode):
+                stats.nfiles += 1
+                stats.osize += st.st_size
+            elif read_special:
+                if stat.S_ISLNK(st.st_mode):
+                    try:
+                        st_target = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=True)
+                    except OSError:
+                        special = False
+                    else:
+                        special = is_special(st_target.st_mode)
+                else:
+                    special = is_special(st.st_mode)
+                if special:
+                    stats.nfiles += 1  # size unknown without reading the special file
             return "+"  # included
         MAX_RETRIES = 10  # count includes the initial try (initial try == "retry 0")
         for retry in range(MAX_RETRIES):
@@ -673,8 +710,14 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         When using ``--stats``, you will get some statistics about how much data was
         added - the "This Archive" deduplicated size there is most interesting as that is
         how much your repository will grow. Please note that the "All archives" stats refer to
-        the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually
-        exclusive because the data is not actually compressed and deduplicated during a dry run.
+        the state after creation.
+
+        When ``--stats`` is used together with ``--dry-run``, only the number of files and the
+        original size are reported. They are computed from file system metadata, without reading
+        the file contents, so a dry run stays fast. As data is not actually read, chunked, and
+        deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read
+        from standard input, from a command's output, or from special files (``--read-special``)
+        are also unknown in a dry run and counted as zero.
 
         The ``--stats`` output also reports the store statistics (lines prefixed with
         "Store"), taken from the storage layer after this run. These cover the backend and
@@ -834,8 +877,6 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         subparser = ArgumentParser(parents=[common_parser], description=self.do_create.__doc__, epilog=create_epilog)
         subparsers.add_subcommand("create", subparser, help="create a backup")
 
-        # note: --dry-run and --stats are mutually exclusive, but we do not want to abort when
-        #  parsing, but rather proceed with the dry-run, but without stats (see run() method).
         subparser.add_argument(
             "-n", "--dry-run", dest="dry_run", action="store_true", help="do not create a backup archive"
         )
diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py
index 15c2f064e8..b3a51ec785 100644
--- a/src/borg/testsuite/archiver/create_cmd_test.py
+++ b/src/borg/testsuite/archiver/create_cmd_test.py
@@ -687,6 +687,39 @@ def test_create_dry_run(archivers, request):
         assert manifest.archives.count() == 0
 
 
+def test_create_dry_run_stats(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    create_regular_file(archiver.input_path, "file2", size=1024 * 20)
+    expected_nfiles = 2
+    if are_hardlinks_supported():
+        os.link(os.path.join(archiver.input_path, "file1"), os.path.join(archiver.input_path, "hardlink1"))
+        expected_nfiles = 3  # as in a real create, each hardlink counts as a file
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    output = cmd(archiver, "create", "--dry-run", "--stats", "test", "input")
+    assert f"Number of files: {expected_nfiles}" in output
+    assert "Original size:" in output
+    assert "Deduplicated size:" not in output
+    # Make sure no archive has been created
+    with Repository(archiver.repository_path) as repository:
+        manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
+        assert manifest.archives.count() == 0
+
+
+def test_create_dry_run_json(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    create_regular_file(archiver.input_path, "file2", size=1024 * 20)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    output = cmd(archiver, "create", "--dry-run", "--json", "test", "input")
+    result = json.loads(output)
+    assert result["dry_run"] is True
+    assert result["stats"]["nfiles"] == 2
+    assert result["stats"]["original_size"] == 1024 * 100
+    assert "archive" not in result
+    assert "repository" in result
+
+
 def test_progress_on(archivers, request):
     archiver = request.getfixturevalue(archivers)
     create_regular_file(archiver.input_path, "file1", size=1024 * 80)

From b893d49154b3b1b349cec2f920677ada8a5a88b9 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 16:04:07 +0200
Subject: [PATCH 122/144] chunkers: release the GIL in the AES chunkers' scan
 kernels

ra_scan/gl_scan/tp_scan are pure C (OpenSSL EVP or AES hw intrinsics,
no Python API calls), but were called with the GIL held, so a scan
span - up to the 8 MiB max chunk size - blocked all other threads,
e.g. the PackWriter store-thread between its GIL-releasing parts.

Wrap the calls in `with nogil` (and declare the externs nogil), the
same pattern fastcdc/buzhash/buzhash64 already use (#10014).

Chunker output and single-thread performance are unchanged. Two
threads chunking concurrently (rabin-aes, evp kernel, arm64): the
wall-clock ratio vs one thread drops from 2.02x (fully serialized)
to 1.13x (parallel).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/chunkers/goldilocks_aes.pyx | 7 +++++--
 src/borg/chunkers/rabin_aes.pyx      | 7 +++++--
 src/borg/chunkers/toeplitz_aes.pyx   | 7 +++++--
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/borg/chunkers/goldilocks_aes.pyx b/src/borg/chunkers/goldilocks_aes.pyx
index 4c86a118c0..2e8d1add53 100644
--- a/src/borg/chunkers/goldilocks_aes.pyx
+++ b/src/borg/chunkers/goldilocks_aes.pyx
@@ -53,7 +53,7 @@ cdef extern from "goldilocks_aes_impl.h":
     void gl_free(GL_CTX *ctx)
     const char *gl_kind(const GL_CTX *ctx)
     uint64_t gl_digest64(const GL_CTX *ctx, const uint8_t *q)
-    int64_t gl_scan(GL_CTX *ctx, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask)
+    int64_t gl_scan(GL_CTX *ctx, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) nogil
 
 
 # --- Goldilocks field helpers (pure Python, used at chunker setup only) ---
@@ -165,7 +165,10 @@ cdef class ChunkerGoldilocksAES(ChunkerPHTE):
             self.ctx = NULL
 
     cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept:
-        return gl_scan(self.ctx, p, n, digest, mask)
+        cdef int64_t r
+        with nogil:
+            r = gl_scan(self.ctx, p, n, digest, mask)
+        return r
 
     cdef uint64_t _digest64(self, const uint8_t *q) noexcept:
         return gl_digest64(self.ctx, q)
diff --git a/src/borg/chunkers/rabin_aes.pyx b/src/borg/chunkers/rabin_aes.pyx
index 953d73c444..7a310c6c67 100644
--- a/src/borg/chunkers/rabin_aes.pyx
+++ b/src/borg/chunkers/rabin_aes.pyx
@@ -64,7 +64,7 @@ cdef extern from "rabin_aes_impl.h":
     void ra_free(RA_CTX *ctx)
     const char *ra_kind(const RA_CTX *ctx)
     uint64_t ra_digest64(const RA_CTX *ctx, const uint8_t *q)
-    int64_t ra_scan(RA_CTX *ctx, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask)
+    int64_t ra_scan(RA_CTX *ctx, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) nogil
 
 
 # --- GF(2)[x] polynomial helpers (pure Python, used at chunker setup only) ---
@@ -234,7 +234,10 @@ cdef class ChunkerRabinAES(ChunkerPHTE):
             self.ctx = NULL
 
     cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept:
-        return ra_scan(self.ctx, p, n, digest, mask)
+        cdef int64_t r
+        with nogil:
+            r = ra_scan(self.ctx, p, n, digest, mask)
+        return r
 
     cdef uint64_t _digest64(self, const uint8_t *q) noexcept:
         return ra_digest64(self.ctx, q)
diff --git a/src/borg/chunkers/toeplitz_aes.pyx b/src/borg/chunkers/toeplitz_aes.pyx
index f9fc7cbb44..084663b4b8 100644
--- a/src/borg/chunkers/toeplitz_aes.pyx
+++ b/src/borg/chunkers/toeplitz_aes.pyx
@@ -57,7 +57,7 @@ cdef extern from "toeplitz_aes_impl.h":
     void tp_free(TP_CTX *ctx)
     const char *tp_kind(const TP_CTX *ctx)
     uint64_t tp_digest64(const TP_CTX *ctx, const uint8_t *q)
-    int64_t tp_scan(TP_CTX *ctx, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask)
+    int64_t tp_scan(TP_CTX *ctx, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) nogil
 
 
 # --- GF(2)[x] helpers (pure Python, used at chunker setup only) ----------
@@ -166,7 +166,10 @@ cdef class ChunkerToeplitzAES(ChunkerPHTE):
             self.ctx = NULL
 
     cdef int64_t _scan(self, const uint8_t *p, size_t n, uint64_t *digest, uint64_t mask) noexcept:
-        return tp_scan(self.ctx, p, n, digest, mask)
+        cdef int64_t r
+        with nogil:
+            r = tp_scan(self.ctx, p, n, digest, mask)
+        return r
 
     cdef uint64_t _digest64(self, const uint8_t *q) noexcept:
         return tp_digest64(self.ctx, q)

From 9cc37123fd4892c02059ad301749174ed2537fe4 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 16:20:26 +0200
Subject: [PATCH 123/144] docs: NetBSD xattr support is implemented, update
 platform feature table, #1332

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/usage/general/file-metadata.rst.inc | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/usage/general/file-metadata.rst.inc b/docs/usage/general/file-metadata.rst.inc
index a01d72d3b6..5bf4fba63c 100644
--- a/docs/usage/general/file-metadata.rst.inc
+++ b/docs/usage/general/file-metadata.rst.inc
@@ -35,11 +35,11 @@ On some platforms additional features are supported:
 +-------------------------+----------+-----------+------------+
 | OpenBSD                 | n/a      | n/a       | Yes (all)  |
 +-------------------------+----------+-----------+------------+
-| NetBSD                  | n/a      | No [2]_   | Yes (all)  |
+| NetBSD                  | n/a      | Yes       | Yes (all)  |
 +-------------------------+----------+-----------+------------+
-| Solaris and derivatives | No [3]_  | No [3]_   | n/a        |
+| Solaris and derivatives | No [2]_  | No [2]_   | n/a        |
 +-------------------------+----------+-----------+------------+
-| Windows (cygwin)        | No [4]_  | No        | No         |
+| Windows (cygwin)        | No [3]_  | No        | No         |
 +-------------------------+----------+-----------+------------+
 
 Other Unix-like operating systems may work as well, but have not been tested yet.
@@ -49,9 +49,8 @@ For example, ntfs-3g on Linux is not able to convey NTFS ACLs.
 
 .. [1] Only "nodump", "immutable", "compressed" and "append" are supported.
     Feature request :issue:`618` for more flags.
-.. [2] Feature request :issue:`1332`
-.. [3] Feature request :issue:`1337`
-.. [4] Cygwin tries to map NTFS ACLs to permissions with varying degrees of success.
+.. [2] Feature request :issue:`1337`
+.. [3] Cygwin tries to map NTFS ACLs to permissions with varying degrees of success.
 
 .. [#acls] The native access control list mechanism of the OS. This normally limits access to
     non-native ACLs. For example, NTFS ACLs are not completely accessible on Linux with ntfs-3g.

From 64d622ca5d9e84ae775d6ea513c5d611b7f1c3c5 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 16:47:22 +0200
Subject: [PATCH 124/144] illumos/Solaris: add xattr support, see #1337

xattrs on these platforms are regular files inside a hidden attribute
directory attached to each file (opened via openat(2) with O_XATTR,
see fsattr(7)), so this can be implemented in pure Python: xattr
names/values map to the names/contents of the files in the attribute
directory.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/ci.yml                 |   6 ++
 docs/usage/general/file-metadata.rst.inc |   2 +-
 src/borg/platform/__init__.py            |  11 ++-
 src/borg/platform/solaris.py             | 101 +++++++++++++++++++++++
 src/borg/platformflags.py                |   1 +
 src/borg/testsuite/xattr_test.py         |  16 ++--
 src/borg/xattr.py                        |   2 +-
 7 files changed, 131 insertions(+), 8 deletions(-)
 create mode 100644 src/borg/platform/solaris.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8de5fb3694..aaa5728f5a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -598,6 +598,12 @@ jobs:
               export TMPDIR=/var/tmp/borg-ci
               mkdir -p "$TMPDIR"
 
+              # show whether xattrs work on the ZFS-backed TMPDIR (they are files in a
+              # hidden per-file attribute directory there, listed via runat(1))
+              touch "$TMPDIR/testfile"
+              /usr/bin/runat "$TMPDIR/testfile" ls -a && echo "*** xattrs supported on $TMPDIR ***"
+              rm "$TMPDIR/testfile"
+
               python3 -m venv .venv
               . .venv/bin/activate
               python -V
diff --git a/docs/usage/general/file-metadata.rst.inc b/docs/usage/general/file-metadata.rst.inc
index 5bf4fba63c..068ee1152b 100644
--- a/docs/usage/general/file-metadata.rst.inc
+++ b/docs/usage/general/file-metadata.rst.inc
@@ -37,7 +37,7 @@ On some platforms additional features are supported:
 +-------------------------+----------+-----------+------------+
 | NetBSD                  | n/a      | Yes       | Yes (all)  |
 +-------------------------+----------+-----------+------------+
-| Solaris and derivatives | No [2]_  | No [2]_   | n/a        |
+| Solaris and derivatives | No [2]_  | Yes       | n/a        |
 +-------------------------+----------+-----------+------------+
 | Windows (cygwin)        | No [3]_  | No        | No         |
 +-------------------------+----------+-----------+------------+
diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py
index 168d4ce705..4ff1b77a07 100644
--- a/src/borg/platform/__init__.py
+++ b/src/borg/platform/__init__.py
@@ -6,7 +6,7 @@
 
 from types import ModuleType
 
-from ..platformflags import is_win32, is_linux, is_freebsd, is_netbsd, is_darwin, is_cygwin, is_haiku
+from ..platformflags import is_win32, is_linux, is_freebsd, is_netbsd, is_darwin, is_cygwin, is_haiku, is_sunos
 
 from .base import ENOATTR
 from .base import SaveFile, sync_dir, fdatasync, safe_fadvise
@@ -59,6 +59,15 @@
     from .posix import get_errno
     from .posix import getosusername
     from . import posix_ug as platform_ug
+elif is_sunos:  # pragma: sunos only
+    from .solaris import listxattr, getxattr, setxattr
+    from .base import acl_get, acl_set
+    from .base import set_flags, get_flags
+    from .base import SyncFile
+    from .posix import process_alive, local_pid_alive
+    from .posix import get_errno
+    from .posix import getosusername
+    from . import posix_ug as platform_ug
 elif not is_win32:  # pragma: posix only
     # Generic code for all other POSIX OSes
     from .base import listxattr, getxattr, setxattr
diff --git a/src/borg/platform/solaris.py b/src/borg/platform/solaris.py
new file mode 100644
index 0000000000..56d8f7ab28
--- /dev/null
+++ b/src/borg/platform/solaris.py
@@ -0,0 +1,101 @@
+"""
+xattr support for illumos / Solaris and derivatives.
+
+On these platforms, the extended attributes of a file are regular files inside a hidden
+attribute directory attached to that file. The attribute directory is opened by giving
+O_XATTR to open(2)/openat(2), see fsattr(7) — xattr names/values map to the names/contents
+of the files in there. There are no xattr namespaces, so names are used verbatim.
+"""
+
+import errno
+import os
+
+from .base import ENOATTR
+
+# CPython exposes os.O_XATTR on Solaris-derived platforms; 0x4000 is its value on illumos
+# and Oracle Solaris (belt and braces in case the os module does not have it).
+O_XATTR = getattr(os, "O_XATTR", 0x4000)
+
+# "Extended system attributes" maintained by the OS inside every attribute directory
+# (e.g. on ZFS) — these are not user-set xattrs, so hide them from listing and refuse
+# to write them.
+SYSATTR_PREFIX = "SUNWattr_"
+
+# Attribute files can be arbitrarily large — refuse to read values bigger than this
+# (the other platforms' xattr support is limited alike, via their Buffer(limit=2**24)).
+XATTR_SIZE_LIMIT = 2**24
+
+
+def _open_attrdir(path, follow_symlinks):
+    # Open the hidden attribute directory of *path* (a bytes path or an open file descriptor).
+    # O_XATTR is only interpreted by openat() when looking up relative to a file descriptor
+    # referring to the file, so for a path, the file itself must be opened first.
+    if isinstance(path, int):
+        return os.open(".", os.O_RDONLY | O_XATTR, dir_fd=path)
+    flags = os.O_RDONLY | os.O_NONBLOCK  # O_NONBLOCK: do not hang on FIFOs
+    if not follow_symlinks:
+        flags |= os.O_NOFOLLOW
+    fd = os.open(path, flags)
+    try:
+        return os.open(".", os.O_RDONLY | O_XATTR, dir_fd=fd)
+    finally:
+        os.close(fd)
+
+
+def listxattr(path, *, follow_symlinks=False):
+    try:
+        dirfd = _open_attrdir(path, follow_symlinks)
+    except OSError as e:
+        if e.errno == errno.ELOOP:
+            # symlinks cannot have extended attributes on this platform
+            return []
+        if e.errno == errno.EINVAL:
+            # open(2): the filesystem does not support extended attributes
+            raise OSError(errno.ENOTSUP, os.strerror(errno.ENOTSUP), path) from None
+        raise
+    try:
+        names = os.listdir(dirfd)
+    finally:
+        os.close(dirfd)
+    return [os.fsencode(name) for name in names if not name.startswith(SYSATTR_PREFIX)]
+
+
+def getxattr(path, name, *, follow_symlinks=False):
+    dirfd = _open_attrdir(path, follow_symlinks)
+    try:
+        try:
+            fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=dirfd)
+        except FileNotFoundError:
+            # no attribute file with that name -> no such xattr
+            raise OSError(ENOATTR, os.strerror(ENOATTR), path) from None
+        try:
+            if os.fstat(fd).st_size > XATTR_SIZE_LIMIT:
+                raise OSError(errno.EFBIG, os.strerror(errno.EFBIG), path)
+            chunks = []
+            while chunk := os.read(fd, 2**20):
+                chunks.append(chunk)
+            return b"".join(chunks)
+        finally:
+            os.close(fd)
+    finally:
+        os.close(dirfd)
+
+
+def setxattr(path, name, value, *, follow_symlinks=False):
+    name_str = os.fsdecode(name)
+    if not name_str or "/" in name_str or "\0" in name_str or name_str in (".", ".."):
+        # such a name cannot be the name of an attribute file
+        raise OSError(errno.EINVAL, os.strerror(errno.EINVAL), path)
+    if name_str.startswith(SYSATTR_PREFIX):
+        raise OSError(errno.EPERM, os.strerror(errno.EPERM), path)
+    dirfd = _open_attrdir(path, follow_symlinks)
+    try:
+        fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, mode=0o644, dir_fd=dirfd)
+        try:
+            mv = memoryview(value)
+            while mv:
+                mv = mv[os.write(fd, mv) :]
+        finally:
+            os.close(fd)
+    finally:
+        os.close(dirfd)
diff --git a/src/borg/platformflags.py b/src/borg/platformflags.py
index cc84f7c7ac..c6f98270b7 100644
--- a/src/borg/platformflags.py
+++ b/src/borg/platformflags.py
@@ -16,6 +16,7 @@
 is_openbsd = sys.platform.startswith("openbsd")
 is_darwin = sys.platform.startswith("darwin")
 is_haiku = sys.platform.startswith("haiku")
+is_sunos = sys.platform.startswith("sunos")  # illumos and Solaris
 
 # MSYS2 (on Windows)
 is_msystem = is_win32 and "MSYSTEM" in os.environ
diff --git a/src/borg/testsuite/xattr_test.py b/src/borg/testsuite/xattr_test.py
index f99aa961c2..4f41588fbc 100644
--- a/src/borg/testsuite/xattr_test.py
+++ b/src/borg/testsuite/xattr_test.py
@@ -4,7 +4,12 @@
 
 from ..platform.xattr import buffer, split_lstring
 from ..xattr import is_enabled, getxattr, setxattr, listxattr, XATTR_FAKEROOT
-from ..platformflags import is_linux
+from ..platformflags import is_linux, is_sunos
+
+# Whether xattrs can be set on a symlink itself:
+# Linux does not allow setting user.* xattrs on symlinks; on illumos/Solaris,
+# symlinks cannot have extended attributes at all.
+symlink_xattrs = not is_linux and not is_sunos
 
 
 @pytest.fixture()
@@ -35,22 +40,22 @@ def test(tempfile_symlink):
     setxattr(tmp_fn, b"user.foo", b"bar")
     setxattr(tmp_fd, b"user.bar", b"foo")
     setxattr(tmp_fn, b"user.empty", b"")
-    if not is_linux:
-        # Linux does not allow setting user.* xattrs on symlinks.
+    if symlink_xattrs:
         setxattr(tmp_lfn, b"user.linkxattr", b"baz")
     assert_equal_se(listxattr(tmp_fn), [b"user.foo", b"user.bar", b"user.empty"])
     assert_equal_se(listxattr(tmp_fd), [b"user.foo", b"user.bar", b"user.empty"])
     assert_equal_se(listxattr(tmp_lfn, follow_symlinks=True), [b"user.foo", b"user.bar", b"user.empty"])
-    if not is_linux:
+    if symlink_xattrs:
         assert_equal_se(listxattr(tmp_lfn), [b"user.linkxattr"])
     assert getxattr(tmp_fn, b"user.foo") == b"bar"
     assert getxattr(tmp_fd, b"user.foo") == b"bar"
     assert getxattr(tmp_lfn, b"user.foo", follow_symlinks=True) == b"bar"
-    if not is_linux:
+    if symlink_xattrs:
         assert getxattr(tmp_lfn, b"user.linkxattr") == b"baz"
     assert getxattr(tmp_fn, b"user.empty") == b""
 
 
+@pytest.mark.skipif(is_sunos, reason="the illumos/Solaris xattr backend does not use the shared buffer")
 def test_listxattr_buffer_growth(tempfile_symlink):
     temp_file, symlink = tempfile_symlink
     tmp_fn = os.fsencode(temp_file.name)
@@ -65,6 +70,7 @@ def test_listxattr_buffer_growth(tempfile_symlink):
     assert len(buffer) > 64
 
 
+@pytest.mark.skipif(is_sunos, reason="the illumos/Solaris xattr backend does not use the shared buffer")
 def test_getxattr_buffer_growth(tempfile_symlink):
     temp_file, symlink = tempfile_symlink
     tmp_fn = os.fsencode(temp_file.name)
diff --git a/src/borg/xattr.py b/src/borg/xattr.py
index 776a946774..eb04149f22 100644
--- a/src/borg/xattr.py
+++ b/src/borg/xattr.py
@@ -1,4 +1,4 @@
-"""A basic extended attributes (xattr) implementation for Linux, FreeBSD and macOS."""
+"""A basic extended attributes (xattr) implementation for Linux, FreeBSD, macOS, NetBSD and illumos/Solaris."""
 
 import errno
 import os

From 0ada4c73b7e4de661467ac76151fca614c7ce22e Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 17:18:32 +0200
Subject: [PATCH 125/144] tests: relax wall-clock bound in read-special-timeout
 test

test_create_read_special_timeout_expired asserted that borg create with
--read-special-timeout=1 returns within 30s. On heavily loaded CI runners
the whole command (incl. remote archiver setup) occasionally takes longer
than that (observed: 37s on a macOS runner), failing the test although
the timeout mechanism worked correctly.

Relax the bound to 300s: it still proves borg gave up soon after the
requested 1s timeout instead of waiting for the 30 minutes default
timeout, but no longer trips over slow runners.
---
 src/borg/testsuite/archiver/create_cmd_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py
index 15c2f064e8..d0e4b072f7 100644
--- a/src/borg/testsuite/archiver/create_cmd_test.py
+++ b/src/borg/testsuite/archiver/create_cmd_test.py
@@ -1037,7 +1037,9 @@ def test_create_read_special_timeout_expired(archivers, request):
         "input",
         exit_code=expected_ec,  # WARNING status: could not back up the fifo.
     )
-    assert time.monotonic() - started < 30  # timed out, no eternal hang
+    # Generous bound for heavily loaded CI runners: proves borg gave up soon after the requested
+    # 1s timeout (and did not e.g. wait for the 30 minutes default timeout).
+    assert time.monotonic() - started < 300
     assert "retry: 1 of " not in out  # timeouts are NOT retried
     listing = cmd(archiver, "list", "test", "--format={path}{NL}")
     assert "input/file1" in listing

From 4dfcf29ad2906df47fc0b4d681a587bdd0ab4ada Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 19:40:41 +0200
Subject: [PATCH 126/144] CI: pin all GitHub Actions to commit SHAs

Tags are mutable. An attacker who compromises an action repository can
move an existing tag to a malicious commit, which then runs in our
workflows with the workflow token and can tamper with build artifacts
before they get attested.

Pin every action to the full commit SHA of the release it currently
resolves to and keep the human-readable version in a trailing comment,
matching the convention already used for psf/black and
peter-evans/create-pull-request.

Dependabot keeps updating SHA-pinned actions, bumping both the SHA and
the version comment.
---
 .github/workflows/backport.yml                |  4 +-
 .github/workflows/black.yaml                  |  2 +-
 .github/workflows/canary-py312-ubuntu2604.yml |  2 +-
 .github/workflows/canary.yml                  | 10 ++--
 .github/workflows/ci.yml                      | 54 +++++++++----------
 .github/workflows/codeql-analysis.yml         | 10 ++--
 .github/workflows/fame.yml                    |  4 +-
 7 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index deab27e150..7f06c99e9c 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -31,8 +31,8 @@ jobs:
           startsWith(github.event.comment.body, '/backport')
         )
     steps:
-        - uses: actions/checkout@v7
+        - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         - name: Create backport pull requests
-          uses: korthout/backport-action@v4
+          uses: korthout/backport-action@2e830a1d0b8269505846ddd407a70876913ad1f8  # v4.6.0
           with:
             label_pattern: '^port/(.+)$'
diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index 33ef1f60f1..94c22ea57d 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - uses: psf/black@87928e6d6761a4a6d22250e1fee5601b3998086e  # 26.5.1
         with:
             version: "~= 24.0"
diff --git a/.github/workflows/canary-py312-ubuntu2604.yml b/.github/workflows/canary-py312-ubuntu2604.yml
index a770915ad8..c4658898da 100644
--- a/.github/workflows/canary-py312-ubuntu2604.yml
+++ b/.github/workflows/canary-py312-ubuntu2604.yml
@@ -30,7 +30,7 @@ jobs:
     - name: Try to set up Python 3.12
       id: setup
       continue-on-error: true
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: '3.12'
 
diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml
index b812d61542..818cee44b7 100644
--- a/.github/workflows/canary.yml
+++ b/.github/workflows/canary.yml
@@ -44,13 +44,13 @@ jobs:
             toxenv: py314-none
 
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       with:
         fetch-depth: 0
         fetch-tags: true
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -125,7 +125,7 @@ jobs:
         shell: msys2 {0}
 
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
 
@@ -134,14 +134,14 @@ jobs:
       # unlocked requirements may resolve to different versions; falls back
       # to (and gets picked up by) the ci.yml cache via the shared prefix.
       - name: Cache pip-built wheels
-        uses: actions/cache@v6
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
         with:
           path: .pip-cache
           key: windows-msys2-pip-canary-${{ hashFiles('pyproject.toml', 'requirements.d/pyinstaller.txt') }}
           restore-keys: |
             windows-msys2-pip-
 
-      - uses: msys2/setup-msys2@v2
+      - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884  # v2.32.0
         with:
           msystem: UCRT64
           update: true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aaa5728f5a..334ef1ca13 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -42,8 +42,8 @@ jobs:
     timeout-minutes: 5
 
     steps:
-    - uses: actions/checkout@v7
-    - uses: astral-sh/ruff-action@v4.1.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+    - uses: astral-sh/ruff-action@278981a28ce3188b1e39527901f38254bf3aac89  # v4.1.0
 
   security:
 
@@ -51,9 +51,9 @@ jobs:
     timeout-minutes: 5
 
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
     - name: Set up Python
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: '3.11'
     - name: Install dependencies
@@ -71,14 +71,14 @@ jobs:
     needs: [lint]
 
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       with:
         # Just fetching one commit is not enough for setuptools-scm, so we fetch all.
         fetch-depth: 0
         fetch-tags: true
 
     - name: Set up Python
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: '3.12'
 
@@ -182,19 +182,19 @@ jobs:
     timeout-minutes: 360
 
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       with:
         # Just fetching one commit is not enough for setuptools-scm, so we fetch all.
         fetch-depth: 0
         fetch-tags: true
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Cache pip
-      uses: actions/cache@v6
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-${{ runner.arch }}-pip-${{ hashFiles('requirements.d/development.lock.txt') }}
@@ -203,7 +203,7 @@ jobs:
             ${{ runner.os }}-${{ runner.arch }}-
 
     - name: Cache tox environments
-      uses: actions/cache@v6
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
       with:
         path: .tox
         key: ${{ runner.os }}-${{ runner.arch }}-tox-${{ matrix.toxenv }}-${{ hashFiles('requirements.d/development.lock.txt', 'pyproject.toml') }}
@@ -352,13 +352,13 @@ jobs:
 
     - name: Attest binaries provenance (${{ matrix.binary }})
       if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
-      uses: actions/attest-build-provenance@v4
+      uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8  # v4.2.2
       with:
         subject-path: 'artifacts/*'
 
     - name: Upload binaries (${{ matrix.binary }})
       if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
       with:
         name: ${{ matrix.binary }}
         path: artifacts/*
@@ -374,7 +374,7 @@ jobs:
 
     - name: Upload test results to Codecov
       if: ${{ !cancelled() && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}
-      uses: codecov/codecov-action@v7
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
       env:
         OS: ${{ runner.os }}
         python: ${{ matrix.python-version }}
@@ -386,7 +386,7 @@ jobs:
 
     - name: Upload coverage to Codecov
       if: ${{ !cancelled() && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}
-      uses: codecov/codecov-action@v7
+      uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
       env:
         OS: ${{ runner.os }}
         python: ${{ matrix.python-version }}
@@ -436,7 +436,7 @@ jobs:
 
     steps:
       - name: Check out repository
-        uses: actions/checkout@v7
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
           fetch-tags: true
@@ -445,7 +445,7 @@ jobs:
       # rsyncs into the VM and back, so wheels built from sdists in one run
       # (most of the VM setup time) are reused by the next one.
       - name: Cache pip-built wheels
-        uses: actions/cache@v6
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
         with:
           path: .pip-cache
           key: ${{ matrix.os }}-${{ matrix.version }}-pip-${{ hashFiles('requirements.d/development.lock.txt') }}
@@ -457,7 +457,7 @@ jobs:
         # a normal boot takes < 1 minute; since 1.4.0 the action bounds
         # waiting for boot itself, this is just an additional safety net.
         timeout-minutes: 15
-        uses: cross-platform-actions/action@v1.4.0
+        uses: cross-platform-actions/action@24ef01df165c76df1ed2b9f9e9212e78dc2fc963  # v1.4.0
         with:
           operating_system: ${{ matrix.os }}
           version: ${{ matrix.version }}
@@ -644,7 +644,7 @@ jobs:
 
       - name: Upload artifacts
         if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: ${{ matrix.artifact_prefix }}
           path: artifacts/*
@@ -652,13 +652,13 @@ jobs:
 
       - name: Attest provenance
         if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
-        uses: actions/attest-build-provenance@v4
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8  # v4.2.2
         with:
           subject-path: 'artifacts/*'
 
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}
-        uses: codecov/codecov-action@v7
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
         env:
           OS: ${{ matrix.os }}
         with:
@@ -669,7 +669,7 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
-        uses: codecov/codecov-action@v7
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
         env:
           OS: ${{ matrix.os }}
         with:
@@ -697,7 +697,7 @@ jobs:
         shell: msys2 {0}
 
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
 
@@ -705,14 +705,14 @@ jobs:
       # all compiled deps from source - incl. building maturin via cargo, just
       # to build the blake3 wheel. Persist the wheels pip builds.
       - name: Cache pip-built wheels
-        uses: actions/cache@v6
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
         with:
           path: .pip-cache
           key: windows-msys2-pip-${{ hashFiles('pyproject.toml', 'requirements.d/pyinstaller.txt') }}
           restore-keys: |
             windows-msys2-pip-
 
-      - uses: msys2/setup-msys2@v2
+      - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884  # v2.32.0
         with:
           msystem: UCRT64
           update: true
@@ -738,7 +738,7 @@ jobs:
           # build sdist and wheel in dist/...
           python -m build
 
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: borg-windows
           path: dist/binary/borg.exe
@@ -753,7 +753,7 @@ jobs:
 
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}
-        uses: codecov/codecov-action@v7
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
         env:
           OS: ${{ runner.os }}
           python: '3.11'
@@ -765,7 +765,7 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
-        uses: codecov/codecov-action@v7
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7.0.0
         env:
           OS: ${{ runner.os }}
           python: '3.11'
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index b4be4306cb..834bffadcc 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -46,16 +46,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v7
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       with:
         # Just fetching one commit is not enough for setuptools-scm, so we fetch all.
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: 3.11
     - name: Cache pip
-      uses: actions/cache@v6
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
@@ -69,7 +69,7 @@ jobs:
         sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v4.37.6
+      uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3  # v4.37.6
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -83,4 +83,4 @@ jobs:
         pip3 install -r requirements.d/development.txt
         pip3 install -ve .
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v4.37.6
+      uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3  # v4.37.6
diff --git a/.github/workflows/fame.yml b/.github/workflows/fame.yml
index cb96c605b4..1baf34394c 100644
--- a/.github/workflows/fame.yml
+++ b/.github/workflows/fame.yml
@@ -42,13 +42,13 @@ jobs:
       pull-requests: write
 
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       with:
         ref: master
         fetch-depth: 0  # git blame needs the whole history
 
     - name: Set up Python
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: '3.14'
 

From daa1944e74fecb5aa53ff00458b3332d60ce8edd Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 19:48:37 +0200
Subject: [PATCH 127/144] CI: add a Dependabot cooldown for GitHub Actions

Malicious releases are usually caught and pulled within days, so waiting
before adopting a new version buys herd immunity for little cost. A
compromised action is the more severe case of the two we consume: it
runs with the workflow token and can tamper with build artifacts before
they get attested.

Mirror the major/minor values already used for pip. default-days
additionally covers patch releases and actions that do not use semver,
which the pip block does not set.

Security updates are not delayed by cooldown.
---
 .github/dependabot.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index e18ae10ccc..b65cdf26b3 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -4,6 +4,11 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      # default-days also covers patch releases and actions not using semver.
+      default-days: 14
+      semver-major-days: 90
+      semver-minor-days: 30
     groups:
       actions:
         patterns:

From 18def9bdebb2471693db88e283479f7e48e736d0 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 20:18:57 +0200
Subject: [PATCH 128/144] docs: FAQ, 2 ways to limit bandwidth, fixes #8838

BORGSTORE_BANDWIDTH works for all backends and in both directions, but
does not shape the traffic. pv on both sides of a ssh ProxyCommand is
smoother and has a separate limit per direction, but only works for
rest:// repositories. Give pros and cons of each.

Also mention BORGSTORE_LATENCY and rclone's own --bwlimit.
---
 docs/faq.rst | 76 +++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 15 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index e4fd87e59a..2c731ebc8e 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -1032,33 +1032,79 @@ code).
 Is there a way to limit bandwidth with Borg?
 --------------------------------------------
 
-Borg has no built-in bandwidth limiting - the ``--remote-ratelimit`` and
-``--upload-ratelimit`` options were removed.
+Borg has no bandwidth limiting option - ``--remote-ratelimit`` and
+``--upload-ratelimit`` were removed. There are 2 ways to do it anyway:
 
-For repositories accessed via ssh, bandwidth can be limited with pipeviewer_:
+Using borgstore's bandwidth limit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Create a wrapper script:  /usr/local/bin/pv-wrapper
+borgstore, which Borg uses for repository access, can limit the transfer rate::
 
-::
+    # 16 Mbit/s == 2 MB/s, 0 (the default) means unlimited:
+    export BORGSTORE_BANDWIDTH=16000000
+
+Pros:
+
+- works for all backends (``sftp://``, ``rest://``, ``rclone:``, ``s3://``, ...).
+- limits both directions.
+- needs no additional software.
+
+Cons:
+
+- the value is given in **bits** per second (easy to confuse with ``pv``, which
+  uses bytes per second).
+- only the object payload is accounted for, protocol overhead (ssh, TLS, http,
+  ...) comes on top, so the real usage is a bit higher than the given rate.
+- it does not shape the traffic: an object is transferred at full speed and
+  Borg then waits until the given rate is reached on average. As Borg transfers
+  rather big objects (pack files are up to 50MB), the connection will be busy
+  for a while and idle afterwards.
+
+There is also ``BORGSTORE_LATENCY`` (in microseconds), which adds a delay per
+backend call.
+
+Using pv on the ssh connection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    #!/bin/sh
+For ``rest://`` repositories, Borg connects via ssh, so the transfer can be
+limited with pipeviewer_. Put a ``pv`` on each side of the connection using an
+ssh ``ProxyCommand`` (this needs ``nc``), e.g. in ``~/.ssh/config``::
+
+    Host borghost
         ## -q, --quiet              do not output any transfer information at all
         ## -L, --rate-limit RATE    limit transfer to RATE bytes per second
-    RATE=307200
-    pv -q -L $RATE  | "$@"
+        ProxyCommand pv -q -L 307200 | nc %h %p | pv -q -L 307200
 
-Add BORG_RSH environment variable to use pipeviewer wrapper script with ssh.
+The first ``pv`` limits the upload, the second one the download, each to RATE
+bytes per second.
 
-::
+Pros:
 
-    export BORG_RSH='/usr/local/bin/pv-wrapper ssh'
+- smoother, as ``pv`` limits the rate of the data stream itself.
+- each direction has its own limit.
+- the ssh protocol overhead is included in the limit.
+- the rate can be changed on the fly::
 
-Now Borg will be bandwidth limited. The nice thing about ``pv`` is that you can
-change rate-limit on the fly:
+      pv -R $(pidof pv) -L 102400
 
-::
+  As the ``ProxyCommand`` above runs 2 ``pv`` processes, ``pidof`` will print 2
+  pids - give ``pv -R`` the pid of the direction you want to change.
+
+Cons:
+
+- only works for ``rest://`` repositories. ``sftp://`` does not run an external
+  ssh command (borgstore uses paramiko and creates the connection itself) and
+  a ``ProxyCommand`` in ``~/.ssh/config`` is not used there.
+- needs ``pv`` and ``nc``.
+
+Using rclone's bandwidth limit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For ``rclone:`` repositories, you can also use rclone's own bandwidth limiting,
+see its ``--bwlimit`` option. rclone picks up options from the environment, so
+you can use::
 
-    pv -R $(pidof pv) -L 102400
+    export RCLONE_BWLIMIT=300k
 
 .. _pipeviewer: https://www.ivarch.com/programs/pv.shtml
 

From 9dd91fcb8bdec950bd4deb98e01917a95e24a124 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 23:49:51 +0200
Subject: [PATCH 129/144] create: do not use ctime for the files cache on
 Windows, fixes #7193

On Windows, os.stat_result.st_ctime[_ns] is the file *creation* time
(CPython literally assigns st_ctime = st_birthtime there), not the
"metadata change time" it is on POSIX systems. It therefore never
changes when a file's contents change.

With the default files cache mode "ctime,size,inode", a file that was
modified in place while keeping its size and inode number was thus
considered unchanged and its new contents were silently not backed up -
only the very first archive was complete.

So, on Windows:

- the default files cache mode is "mtime,size,inode" now
- an explicitly given ctime based mode warns and uses the corresponding
  mtime based mode instead (ctime,size,inode -> mtime,size,inode,
  ctime,size -> mtime,size, rechunk,ctime -> rechunk,mtime)

This is the same approach as already used for --files-changed.

Note: the files cache always memorizes both ctime and mtime, so this
does not invalidate existing files cache entries.
---
 docs/changes.rst                              |  6 ++++
 docs/usage/create.rst.inc                     | 15 ++++++---
 src/borg/archiver/create_cmd.py               | 32 ++++++++++++++++---
 src/borg/constants.py                         |  8 +++--
 src/borg/helpers/__init__.py                  |  1 +
 src/borg/helpers/parseformat.py               | 17 ++++++++++
 .../testsuite/archiver/create_cmd_test.py     | 21 +++++++++++-
 .../testsuite/helpers/parseformat_test.py     | 30 +++++++++++++++++
 8 files changed, 118 insertions(+), 12 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index b74f07c795..5ccd15cc17 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -253,6 +253,12 @@ New features:
 
 Fixes:
 
+- create: do not use ctime for the files cache on Windows, #7193.
+
+  ctime is the file *creation* time on Windows, so a ctime based files cache mode
+  did not notice content changes of files that kept their size and inode number.
+  The default is ``mtime,size,inode`` there now and an explicitly given ctime based
+  mode warns and uses the corresponding mtime based mode.
 - re-add XXH64 to read borg 1.x integrity data, #9935
 - list: add {blake3} format key, #9984
 - support date: archive patterns for --from-borg1, #9949
diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc
index b914e9f204..3f87001085 100644
--- a/docs/usage/create.rst.inc
+++ b/docs/usage/create.rst.inc
@@ -91,7 +91,7 @@ borg create
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |                                                       | ``--sparse``                                      | detect sparse holes in input (supported only by fixed chunker)                                                                                                                                                                                 |
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--files-cache MODE``                            | operate files cache in MODE. default: ctime,size,inode                                                                                                                                                                                         |
+    |                                                       | ``--files-cache MODE``                            | operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there).                                                                                                              |
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |                                                       | ``--files-changed MODE``                          | specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).                                                                             |
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
@@ -169,7 +169,7 @@ borg create
         --noacls                  do not read and store ACLs into archive
         --noxattrs                do not read and store xattrs into archive
         --sparse                  detect sparse holes in input (supported only by fixed chunker)
-        --files-cache MODE        operate files cache in MODE. default: ctime,size,inode
+        --files-cache MODE        operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there).
         --files-changed MODE      specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).
         --read-special            open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.
         --read-special-timeout SECONDS    when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds.
@@ -220,8 +220,8 @@ the files cache.
 
 This comparison can operate in different modes as given by ``--files-cache``:
 
-- ctime,size,inode (default)
-- mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4)
+- ctime,size,inode (default on POSIX systems)
+- mtime,size,inode (default on Windows)
 - ctime,size (ignore the inode number)
 - mtime,size (ignore the inode number)
 - rechunk,ctime (all files are considered modified - rechunk, cache ctime)
@@ -249,6 +249,13 @@ ctime vs. mtime: safety vs. speed
   it had before a content change happened. This can be used maliciously as well as
   well-meant, but in both cases mtime-based cache modes can be problematic.
 
+On Windows, ctime is the file *creation* time, not the "metadata change time" it is
+on POSIX systems. A ctime based mode would therefore not notice content changes of a
+file that keeps its size and inode number, so borg defaults to ``mtime,size,inode``
+there. If a ctime based mode is given explicitly on Windows, borg warns and uses the
+corresponding mtime based mode instead: ctime,size,inode -> mtime,size,inode,
+ctime,size -> mtime,size, rechunk,ctime -> rechunk,mtime.
+
 The ``--files-changed`` option controls how Borg detects if a file has changed during backup:
  - ctime (default on POSIX): Use ctime to detect changes. This is the safest option.
    Not supported on Windows (ctime is file creation time there).
diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py
index 1727ca14d2..748dbe3be9 100644
--- a/src/borg/archiver/create_cmd.py
+++ b/src/borg/archiver/create_cmd.py
@@ -16,7 +16,8 @@
 from ..cache import Cache
 from ..constants import *  # NOQA
 from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec
-from ..helpers import archivename_validator, FilesCacheMode, octal_int, nonnegative_seconds
+from ..helpers import archivename_validator, FilesCacheMode, files_cache_mode_no_ctime
+from ..helpers import octal_int, nonnegative_seconds
 from ..helpers import eval_escapes
 from ..helpers import timestamp, archive_ts_now
 from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify
@@ -50,6 +51,19 @@ def do_create(self, args, repository, manifest):
             read_special_timeout = READ_SPECIAL_TIMEOUT_DEFAULT
         if read_special_timeout == 0:
             read_special_timeout = None  # wait forever
+        if is_win32:
+            # st_ctime is the file *creation* time on Windows, not the "metadata change time",
+            # so a ctime based files cache mode would not detect content changes of a file that
+            # keeps its size and inode number. Use the mtime based equivalent instead, see #7193.
+            # note: this must happen before the Cache is created (it gets the files cache mode).
+            files_cache_mode, changed = files_cache_mode_no_ctime(args.files_cache_mode)
+            if changed:
+                self.print_warning(
+                    "--files-cache=ctime,... is not supported on Windows "
+                    "(ctime is file creation time, not change time). Using mtime instead.",
+                    wc=None,
+                )
+                args.files_cache_mode = files_cache_mode
         key = manifest.key
         matcher = PatternMatcher(fallback=True)
         matcher.add_inclexcl(args.patterns)
@@ -619,8 +633,8 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
 
         This comparison can operate in different modes as given by ``--files-cache``:
 
-        - ctime,size,inode (default)
-        - mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4)
+        - ctime,size,inode (default on POSIX systems)
+        - mtime,size,inode (default on Windows)
         - ctime,size (ignore the inode number)
         - mtime,size (ignore the inode number)
         - rechunk,ctime (all files are considered modified - rechunk, cache ctime)
@@ -648,6 +662,13 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
           it had before a content change happened. This can be used maliciously as well as
           well-meant, but in both cases mtime-based cache modes can be problematic.
 
+        On Windows, ctime is the file *creation* time, not the "metadata change time" it is
+        on POSIX systems. A ctime based mode would therefore not notice content changes of a
+        file that keeps its size and inode number, so borg defaults to ``mtime,size,inode``
+        there. If a ctime based mode is given explicitly on Windows, borg warns and uses the
+        corresponding mtime based mode instead: ctime,size,inode -> mtime,size,inode,
+        ctime,size -> mtime,size, rechunk,ctime -> rechunk,mtime.
+
         The ``--files-changed`` option controls how Borg detects if a file has changed during backup:
          - ctime (default on POSIX): Use ctime to detect changes. This is the safest option.
            Not supported on Windows (ctime is file creation time there).
@@ -971,8 +992,9 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
             dest="files_cache_mode",
             action=Highlander,
             type=FilesCacheMode,
-            default=FILES_CACHE_MODE_UI_DEFAULT,
-            help="operate files cache in MODE. default: %s" % FILES_CACHE_MODE_UI_DEFAULT,
+            default=FILES_CACHE_MODE_UI_DEFAULT_WIN32 if is_win32 else FILES_CACHE_MODE_UI_DEFAULT_POSIX,
+            help="operate files cache in MODE. default: %s (on Windows: %s, because ctime is "
+            "file creation time there)." % (FILES_CACHE_MODE_UI_DEFAULT_POSIX, FILES_CACHE_MODE_UI_DEFAULT_WIN32),
         )
         fs_group.add_argument(
             "--files-changed",
diff --git a/src/borg/constants.py b/src/borg/constants.py
index 6103eb630e..fb8483fb8b 100644
--- a/src/borg/constants.py
+++ b/src/borg/constants.py
@@ -177,8 +177,12 @@
 # normal on-disk data, allocated (but not written, all zeros), not allocated hole (all zeros)
 CH_DATA, CH_ALLOC, CH_HOLE = 0, 1, 2
 
-# operating mode of the files cache (for fast skipping of unchanged files)
-FILES_CACHE_MODE_UI_DEFAULT = "ctime,size,inode"  # default for "borg create" command (CLI UI)
+# operating mode of the files cache (for fast skipping of unchanged files).
+# note: on Windows, os.stat_result.st_ctime[_ns] is the file *creation* time, not the
+# "metadata change time" it is on POSIX systems. A ctime based mode would not detect
+# content changes of a file that keeps its size and inode number there. See #7193.
+FILES_CACHE_MODE_UI_DEFAULT_POSIX = "ctime,size,inode"  # default for "borg create" command (CLI UI)
+FILES_CACHE_MODE_UI_DEFAULT_WIN32 = "mtime,size,inode"  # same, but on Windows
 FILES_CACHE_MODE_DISABLED = "d"  # most borg commands do not use the files cache at all (disable)
 
 # account for clocks being slightly out-of-sync, timestamps granularity.
diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py
index 7a0fdbccf5..898e92a59a 100644
--- a/src/borg/helpers/__init__.py
+++ b/src/borg/helpers/__init__.py
@@ -38,6 +38,7 @@
     CompressionSpec,
     ChunkerParams,
     FilesCacheMode,
+    files_cache_mode_no_ctime,
     partial_format,
     DatetimeWrapper,
 )
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index fe91fc7cd8..d0c423b80c 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -429,6 +429,23 @@ def FilesCacheMode(s):
     return mode
 
 
+# ctime based files cache modes and their mtime based equivalent, see #7193.
+FILES_CACHE_MODE_CTIME_TO_MTIME = {"cis": "ims", "cs": "ms", "cr": "mr"}
+
+
+def files_cache_mode_no_ctime(mode):
+    """
+    Return (new_mode, changed): <mode> with ctime replaced by mtime.
+
+    <mode> may be given in long form ("ctime,size,inode") or in short form ("cis"),
+    the returned mode is always the short form, as FilesCacheMode() gives it.
+    "changed" tells whether a ctime -> mtime replacement actually happened.
+    """
+    mode = FilesCacheMode(mode)  # long form -> short form (idempotent for the short form)
+    new_mode = FILES_CACHE_MODE_CTIME_TO_MTIME.get(mode, mode)
+    return new_mode, new_mode != mode
+
+
 def partial_format(format, mapping):
     """
     Apply format.format_map(mapping) while preserving unknown keys
diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py
index d0e4b072f7..22a712761b 100644
--- a/src/borg/testsuite/archiver/create_cmd_test.py
+++ b/src/borg/testsuite/archiver/create_cmd_test.py
@@ -741,7 +741,7 @@ def test_create_invalid_tags(archivers, request):
 
 
 @pytest.mark.skipif(
-    is_win32, reason="ctime attribute is file creation time on Windows"
+    is_win32, reason="ctime is file creation time on Windows, ctime cache modes fall back to mtime there, see #7193"
 )  # see https://docs.python.org/3/library/os.html#os.stat_result.st_ctime
 def test_file_status_cs_cache_mode(archivers, request):
     archiver = request.getfixturevalue(archivers)
@@ -760,6 +760,25 @@ def test_file_status_cs_cache_mode(archivers, request):
     assert "M input/file1" in output
 
 
+@pytest.mark.skipif(not is_win32, reason="Windows-only: ctime is file creation time there, see #7193")
+def test_files_cache_ctime_fallback_win32(archivers, request):
+    """test that a ctime based --files-cache mode warns and falls back to the mtime based mode on Windows"""
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file1", contents=b"123")
+    granularity_sleep()  # file2 must have newer timestamps than file1
+    create_regular_file(archiver.input_path, "file2", size=10)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    # note: cmd() asserts rc == 0, so this also covers that the warning does not change the exit code.
+    output = cmd(archiver, "create", "test", "input", "--list", "--files-cache=ctime,size")
+    assert "ctime is file creation time" in output
+    granularity_sleep()  # the rewrite below must get a newer mtime
+    # rewrite in place: same size, same inode, same creation time (== st_ctime on Windows),
+    # only the mtime changes. Without the fallback, this file would be considered unchanged.
+    create_regular_file(archiver.input_path, "file1", contents=b"321")
+    output = cmd(archiver, "create", "test", "input", "--list", "--files-cache=ctime,size")
+    assert "M input/file1" in output
+
+
 def test_files_changed_modes(archivers, request):
     """test that all --files-changed modes are accepted and work"""
     archiver = request.getfixturevalue(archivers)
diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py
index fa95f8e8d4..0e1d7cd713 100644
--- a/src/borg/testsuite/helpers/parseformat_test.py
+++ b/src/borg/testsuite/helpers/parseformat_test.py
@@ -30,6 +30,7 @@
     swidth_slice,
     eval_escapes,
     ChunkerParams,
+    files_cache_mode_no_ctime,
     get_size_units,
     normalize_local_path,
     ArchiveFormatter,
@@ -915,6 +916,35 @@ def test_invalid_chunkerparams(invalid_chunker_params):
         ChunkerParams(invalid_chunker_params)
 
 
+@pytest.mark.parametrize(
+    "mode, expected_mode, expected_changed",
+    [
+        # ctime based modes get replaced by their mtime based equivalent:
+        ("ctime,size,inode", "ims", True),
+        ("cis", "ims", True),
+        ("ctime,size", "ms", True),
+        ("cs", "ms", True),
+        ("rechunk,ctime", "mr", True),
+        ("cr", "mr", True),
+        # everything else stays as it is (but is normalized to the short form):
+        ("mtime,size,inode", "ims", False),
+        ("ims", "ims", False),
+        ("mtime,size", "ms", False),
+        ("rechunk,mtime", "mr", False),
+        ("disabled", "d", False),
+        ("d", "d", False),
+    ],
+)
+def test_files_cache_mode_no_ctime(mode, expected_mode, expected_changed):
+    assert files_cache_mode_no_ctime(mode) == (expected_mode, expected_changed)
+
+
+def test_files_cache_mode_ui_default():
+    # borg create must not default to a ctime based files cache mode on Windows, see #7193.
+    assert FILES_CACHE_MODE_UI_DEFAULT_POSIX == "ctime,size,inode"
+    assert FILES_CACHE_MODE_UI_DEFAULT_WIN32 == "mtime,size,inode"
+
+
 @pytest.mark.parametrize(
     "env_value, expect_newlines",
     [

From 527fc57567732b467597438ff9ec046ac6b18e6b Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Fri, 14 Aug 2026 23:54:11 +0200
Subject: [PATCH 130/144] compression: cap the default zstd MT workers at 4 for
 chunks

The BORG_ZSTD_MT_WORKERS default was the cpu count, but a chunk only
yields ceil(size / 512KiB) compression jobs - 4 for the 2MiB chunks the
default chunker aims at - so most of a big thread pool gets no work,
while the pool is still created and torn down again for every chunk.

Measured at the chunk level on a 12-core machine (fastcdc 512KiB..8MiB
chunks, real data), 4 workers beat 12 on every corpus at the default
zstd,-4: +13% (source code) .. +37% (VM image) big-chunk throughput.
A sweep over 1/2/4/6/8/12 workers, bucketed by jobs-per-chunk, shows
the throughput peak tracking the job count (2-job chunks peak at 2
workers, 4-job chunks at 4) and 12 workers winning nowhere: beyond the
job count, extra threads only add startup overhead.

export-tar compresses one long stream through a single pool using
libzstd's default (large) job size, so it keeps the cpu count as its
default via the new stream=True parameter. An explicitly set
BORG_ZSTD_MT_WORKERS still overrides both defaults, e.g. for chunker
configurations producing much bigger chunks.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/borg/archiver/help_cmd.py       | 11 +++++++-
 src/borg/archiver/tar_cmds.py       |  2 +-
 src/borg/compress.pyi               |  2 +-
 src/borg/compress.pyx               | 41 ++++++++++++++++++++---------
 src/borg/testsuite/compress_test.py | 15 ++++++-----
 5 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/src/borg/archiver/help_cmd.py b/src/borg/archiver/help_cmd.py
index ce34b6c06a..28980f7db6 100644
--- a/src/borg/archiver/help_cmd.py
+++ b/src/borg/archiver/help_cmd.py
@@ -764,15 +764,24 @@ class HelpMixIn:
                 0 means "always multi-threaded", a very large value effectively disables multi-threading.
             BORG_ZSTD_MT_WORKERS
                 When set to a numeric value, use that many threads to zstd-compress a single chunk
-                (default: the cpu count). 0 or 1 means single-threaded compression.
+                (default: the cpu count, but at most 4). 0 or 1 means single-threaded compression.
                 Only relevant when compressing with ``zstd``.
                 Chunks below 768KiB are always compressed single-threaded: libzstd will not use a
                 compression job smaller than 512KiB, so a small chunk gets split very unevenly and
                 multi-threading it would be slower than not doing it at all.
+                The default is capped at 4 because a chunk of the size the default chunker aims at
+                (2MiB) splits into just 4 such jobs: threads beyond that get (nearly) no work, but
+                the whole thread pool is created again for every chunk. Measured on a 12-core
+                machine, 4 threads beat 12 on every test corpus at the default ``zstd,-4``
+                (+13% .. +37%). Raising the value only pays off if you configured the chunker
+                for much bigger chunks. ``borg export-tar`` compresses one long stream instead of
+                separate chunks and always defaults to the cpu count.
                 Multi-threading trades a little compression ratio for speed (measured at ``zstd,3``:
                 +0.05% archive size for 1MiB chunks, +0.64% for 8MiB ones, more at higher levels), and
                 it uses more cpu time in total to reduce the wallclock time. Set it to 1 if you would
                 rather have the smaller archive, or if borg has to share the cpu with other work.
+                Single-threaded can even be faster on data zstd races through anyway, e.g.
+                already-compressed/incompressible data or long-repeat data like VM images.
             BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL
                 Select the scan kernel the ``fastcdc`` / ``buzhash64`` chunker uses (default:
                 ``scalar``, the plain sequential loop). Accepted values are ``avx512``, ``avx2``,
diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py
index f3fd0a8dd5..f4a49ed627 100644
--- a/src/borg/archiver/tar_cmds.py
+++ b/src/borg/archiver/tar_cmds.py
@@ -188,7 +188,7 @@ def create_zstd_filter(stream, stream_close, decompress):
     if decompress:
         zstream = zstd.ZstdFile(stream, "rb")
     else:
-        workers = get_zstd_mt_workers()
+        workers = get_zstd_mt_workers(stream=True)
         if workers > 1:
             params = zstd.CompressionParameter
             options = {params.compression_level: ZSTD_TAR_LEVEL, params.nb_workers: workers}
diff --git a/src/borg/compress.pyi b/src/borg/compress.pyi
index d11b7bff0f..e3575840ea 100644
--- a/src/borg/compress.pyi
+++ b/src/borg/compress.pyi
@@ -4,7 +4,7 @@ ZSTD_JOB_SIZE_MIN: int
 ZSTD_MT_MIN_SIZE: int
 
 def get_compressor(name: str, **kwargs) -> Any: ...
-def get_zstd_mt_workers() -> int: ...
+def get_zstd_mt_workers(stream: bool = ...) -> int: ...
 
 class Compressor:
     def __init__(self, name: Any = ..., **kwargs) -> None: ...
diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx
index c31014dcb4..766b7ab756 100644
--- a/src/borg/compress.pyx
+++ b/src/borg/compress.pyx
@@ -53,17 +53,30 @@ ZSTD_JOB_SIZE_MIN = 512 * 1024
 # 768KiB (= 1.5 jobs) keeps some margin over break-even, as the overhead is machine dependent.
 ZSTD_MT_MIN_SIZE = 768 * 1024
 
-_zstd_mt_workers = None
+_zstd_mt_workers = None  # cached (chunk workers, stream workers)
 
 
-def get_zstd_mt_workers():
-    """How many threads libzstd may use to compress a single chunk.
+def get_zstd_mt_workers(stream=False):
+    """How many threads libzstd may use to compress a single chunk (or stream).
 
-    Defaults to the cpu count, BORG_ZSTD_MT_WORKERS overrides it. 0 or 1 means
-    single-threaded compression, which also avoids the small loss of compression ratio
-    that splitting a chunk into jobs causes (measured at zstd,3: +0.05% for a 1MiB chunk,
-    +0.64% for an 8MiB one; higher levels lose a bit more as they rely on longer match
-    history).
+    BORG_ZSTD_MT_WORKERS overrides the defaults below (for chunks and streams alike).
+    0 or 1 means single-threaded compression, which also avoids the small loss of
+    compression ratio that splitting a chunk into jobs causes (measured at zstd,3:
+    +0.05% for a 1MiB chunk, +0.64% for an 8MiB one; higher levels lose a bit more as
+    they rely on longer match history).
+
+    For chunks (stream=False), the default is the cpu count, but at most 4: a chunk
+    only yields ceil(size / ZSTD_JOB_SIZE_MIN) jobs - 4 for the 2 MiB chunks the
+    default chunker aims at - so threads beyond that get (nearly) no work, while the
+    whole thread pool is still created and torn down again for every single chunk.
+    Measured on a 12-core machine, 4 workers beat 12 on every corpus tested at the
+    default zstd,-4, by +13% (source code) .. +37% (VM image) big-chunk throughput;
+    higher levels showed smaller differences, but no clear win for 12 anywhere.
+    Raise the value if you configured the chunker for much bigger chunks.
+
+    For long streams (stream=True, used by export-tar), the default is the cpu count:
+    one pool with libzstd's default (large) job size compresses the whole stream, so
+    there are enough jobs and the pool overhead is paid only once.
 
     This is called for every chunk, so the result is cached. The env var is evaluated on
     first use rather than at import time, so that an invalid value is reported via borg's
@@ -72,17 +85,19 @@ def get_zstd_mt_workers():
     global _zstd_mt_workers
     if _zstd_mt_workers is None:
         value = os.environ.get("BORG_ZSTD_MT_WORKERS")
+        cpus = os.cpu_count() or 1
         if value is None:
-            workers = os.cpu_count() or 1
+            workers = (min(cpus, 4), cpus)
         else:
             try:
-                workers = int(value)
+                configured = int(value)
             except ValueError:
                 raise Error(f"BORG_ZSTD_MT_WORKERS must be an integer, but is: {value!r}") from None
-            if workers < 0:
-                raise Error(f"BORG_ZSTD_MT_WORKERS must not be negative, but is: {workers}")
+            if configured < 0:
+                raise Error(f"BORG_ZSTD_MT_WORKERS must not be negative, but is: {configured}")
+            workers = (configured, configured)
         _zstd_mt_workers = workers
-    return _zstd_mt_workers
+    return _zstd_mt_workers[1 if stream else 0]
 
 cdef extern from "lz4.h":
     int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
diff --git a/src/borg/testsuite/compress_test.py b/src/borg/testsuite/compress_test.py
index 77981d94f3..73318fe0a9 100644
--- a/src/borg/testsuite/compress_test.py
+++ b/src/borg/testsuite/compress_test.py
@@ -339,17 +339,20 @@ def test_robj_specific_obfuscation(data_length, expected_padding, robj_type):
 
 
 def test_zstd_mt_workers_from_env(monkeypatch):
-    def workers_for(env_value):
+    def workers_for(env_value, stream=False):
         monkeypatch.setattr("borg.compress._zstd_mt_workers", None)  # drop the cache
         if env_value is None:
             monkeypatch.delenv("BORG_ZSTD_MT_WORKERS", raising=False)
         else:
             monkeypatch.setenv("BORG_ZSTD_MT_WORKERS", env_value)
-        return get_zstd_mt_workers()
-
-    assert workers_for(None) == (os.cpu_count() or 1)
-    for value, expected in [("0", 0), ("1", 1), ("4", 4)]:
-        assert workers_for(value) == expected
+        return get_zstd_mt_workers(stream=stream)
+
+    cpus = os.cpu_count() or 1
+    assert workers_for(None) == min(cpus, 4)  # per-chunk default is capped
+    assert workers_for(None, stream=True) == cpus  # stream default is not
+    for value, expected in [("0", 0), ("1", 1), ("4", 4), ("12", 12)]:
+        assert workers_for(value) == expected  # the env var is not capped
+        assert workers_for(value, stream=True) == expected
     for invalid in ["", "yes", "4x", "1.5", "-1"]:
         with pytest.raises(Error):
             workers_for(invalid)

From 9ef369534e934a8ed5517c7d4cbb0f4559d1a57c Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 15 Aug 2026 00:23:01 +0200
Subject: [PATCH 131/144] tests: keep pytest out of the chunkers testsuite
 package __init__

borg.selftest imports the *_self_test.py modules from
src/borg/testsuite/chunkers/, so that package's __init__ runs on every
borg invocation - and a normal borg install has no pytest. Since
8f2de9e80 the __init__ had a hard "import pytest", needed only by the
chunker_with_kernel() helper, so a non-editable install without the dev
dependencies died at startup:

    File "borg/archiver/__init__.py", line 51, in <module>
        from ..selftest import selftest
    File "borg/selftest.py", line 24, in <module>
        from .testsuite.chunkers.buzhash_self_test import ChunkerTestCase
    File "borg/testsuite/chunkers/__init__.py", line 4, in <module>
        import pytest
    ModuleNotFoundError: No module named 'pytest'

chunker_with_kernel() moves into a separate pytest_helpers module that
only the pytest-based tests import, so the self-test modules can be
imported without pytest again.

The guard in selftest() could not catch this: "pytest" not in
dir(module) only inspects the leaf self-test module, never a package
__init__ it imports - as its own comment says.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/borg/testsuite/chunkers/__init__.py       | 24 +++--------------
 src/borg/testsuite/chunkers/buzhash64_test.py |  3 ++-
 src/borg/testsuite/chunkers/fastcdc_test.py   |  3 ++-
 .../testsuite/chunkers/phte_chunkers_test.py  |  3 ++-
 src/borg/testsuite/chunkers/pytest_helpers.py | 26 +++++++++++++++++++
 5 files changed, 36 insertions(+), 23 deletions(-)
 create mode 100644 src/borg/testsuite/chunkers/pytest_helpers.py

diff --git a/src/borg/testsuite/chunkers/__init__.py b/src/borg/testsuite/chunkers/__init__.py
index 7dd2d7ac9d..a48d12a3eb 100644
--- a/src/borg/testsuite/chunkers/__init__.py
+++ b/src/borg/testsuite/chunkers/__init__.py
@@ -1,31 +1,15 @@
+# Note: this is imported by borg.selftest via the *_self_test.py modules in this package,
+#       so it must not import pytest - a normal borg install does not have it.
+#       Helpers that need pytest belong into pytest_helpers.py.
+
 import os
 import tempfile
 
-import pytest
-
 from borg.constants import *  # noqa
 
 from ...chunkers import has_seek_hole
 
 
-def chunker_with_kernel(monkeypatch, envvar, kernel, make):
-    """Build a chunker with <envvar> set to <kernel>, or skip if it cannot run here.
-
-    Which kernels exist depends on the build (compiler support) and on the CPU, so a
-    kernel that cannot be selected is not a failure - the test is skipped, carrying
-    the reason the chunker gave, so `pytest -rs` shows which kernels a machine covered.
-    Requesting an unusable kernel raising (rather than falling back) is itself tested,
-    see test_kernel_env_rejects_unusable.
-    """
-    monkeypatch.setenv(envvar, kernel)
-    try:
-        chunker = make()
-    except ValueError as err:
-        pytest.skip(str(err))
-    assert chunker.kernel == kernel  # we must have got what we asked for, not a fallback
-    return chunker
-
-
 def cf(chunks):
     """Chunk filter."""
 
diff --git a/src/borg/testsuite/chunkers/buzhash64_test.py b/src/borg/testsuite/chunkers/buzhash64_test.py
index 66adcfc701..92e21e1eea 100644
--- a/src/borg/testsuite/chunkers/buzhash64_test.py
+++ b/src/borg/testsuite/chunkers/buzhash64_test.py
@@ -5,7 +5,8 @@
 
 import pytest
 
-from . import cf, cf_expand, chunker_with_kernel
+from . import cf, cf_expand
+from .pytest_helpers import chunker_with_kernel
 from ...chunkers import ChunkerBuzHash64
 from ...chunkers.buzhash64 import buzhash64_get_table
 from ...constants import *  # NOQA
diff --git a/src/borg/testsuite/chunkers/fastcdc_test.py b/src/borg/testsuite/chunkers/fastcdc_test.py
index bb8bdb6a4b..c5eef568f3 100644
--- a/src/borg/testsuite/chunkers/fastcdc_test.py
+++ b/src/borg/testsuite/chunkers/fastcdc_test.py
@@ -5,7 +5,8 @@
 
 import pytest
 
-from . import cf, cf_expand, chunker_with_kernel
+from . import cf, cf_expand
+from .pytest_helpers import chunker_with_kernel
 from ...chunkers import ChunkerFastCDC, get_chunker
 from ...chunkers.fastcdc import fastcdc_get_gear_table
 from ...constants import *  # NOQA
diff --git a/src/borg/testsuite/chunkers/phte_chunkers_test.py b/src/borg/testsuite/chunkers/phte_chunkers_test.py
index dda8354e2b..7279f8890c 100644
--- a/src/borg/testsuite/chunkers/phte_chunkers_test.py
+++ b/src/borg/testsuite/chunkers/phte_chunkers_test.py
@@ -14,7 +14,8 @@
 
 import pytest
 
-from . import cf, cf_expand, chunker_with_kernel
+from . import cf, cf_expand
+from .pytest_helpers import chunker_with_kernel
 from ...chunkers import ChunkerRabinAES, ChunkerGoldilocksAES, ChunkerToeplitzAES, get_chunker
 from ...constants import *  # NOQA
 from ...helpers import hex_to_bin
diff --git a/src/borg/testsuite/chunkers/pytest_helpers.py b/src/borg/testsuite/chunkers/pytest_helpers.py
new file mode 100644
index 0000000000..b1f8d31900
--- /dev/null
+++ b/src/borg/testsuite/chunkers/pytest_helpers.py
@@ -0,0 +1,26 @@
+"""Chunker test helpers that require pytest.
+
+These must not live in this package's __init__.py: borg.selftest imports the *_self_test.py
+modules from this package, so the package __init__ runs on every borg invocation - and a
+normal borg install has no pytest.
+"""
+
+import pytest
+
+
+def chunker_with_kernel(monkeypatch, envvar, kernel, make):
+    """Build a chunker with <envvar> set to <kernel>, or skip if it cannot run here.
+
+    Which kernels exist depends on the build (compiler support) and on the CPU, so a
+    kernel that cannot be selected is not a failure - the test is skipped, carrying
+    the reason the chunker gave, so `pytest -rs` shows which kernels a machine covered.
+    Requesting an unusable kernel raising (rather than falling back) is itself tested,
+    see test_kernel_env_rejects_unusable.
+    """
+    monkeypatch.setenv(envvar, kernel)
+    try:
+        chunker = make()
+    except ValueError as err:
+        pytest.skip(str(err))
+    assert chunker.kernel == kernel  # we must have got what we asked for, not a fallback
+    return chunker

From d8abb7492acfb6de1ef57ef70162ea4c3fd9133b Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 15 Aug 2026 00:43:46 +0200
Subject: [PATCH 132/144] CI: add big-endian (s390x) test under qemu emulation

All our CI machines are little-endian, so the big-endian code paths in the
native code (e.g. the __builtin_bswap64 calls in the chunker kernels) are
never executed and the architecture independence of the repository format
is not tested at all.

Build and test borg on s390x in an emulated container, running the tests of
the endianness sensitive parts (chunkers, crypto, compression, hashindex,
item, repository format).

Additionally run a cross-architecture interoperability test: a repository
written on the big-endian side is checked, extracted and compared on the
little-endian side and the other way round.  Both sides also have to produce
identical chunk ids for the same data - if they did not, the archives would
still be correct, but deduplication between machines of different endianness
would silently not work.

Emulating everything is slow (~35min), so this does not run for every pull
request, but only for pull requests touching native or format relevant code,
plus weekly and on demand.
---
 .github/workflows/bigendian.yml | 200 +++++++++++++++++++++++++
 scripts/endian_interop_test.py  | 257 ++++++++++++++++++++++++++++++++
 2 files changed, 457 insertions(+)
 create mode 100644 .github/workflows/bigendian.yml
 create mode 100644 scripts/endian_interop_test.py

diff --git a/.github/workflows/bigendian.yml b/.github/workflows/bigendian.yml
new file mode 100644
index 0000000000..be8d18d0b1
--- /dev/null
+++ b/.github/workflows/bigendian.yml
@@ -0,0 +1,200 @@
+# Big-endian test: build and test borg on s390x (big-endian) under qemu
+# user-mode emulation, and check that a repository written on a big-endian
+# machine can be read on a little-endian one and the other way round.
+#
+# Why: borg's repository format is architecture independent and the native code
+# has explicit big-endian code paths (e.g. the __builtin_bswap64 calls in the
+# chunker kernels), but every other machine we test on is little-endian, so
+# those code paths are never executed and a bug in them would only be found by
+# the users of s390x, some ppc64 and some mips machines.
+#
+# Everything inside the container is emulated instruction by instruction, so
+# this is slow. Therefore it does not run for every pull request, but only when
+# native code or format relevant code was touched (plus weekly and on demand).
+
+name: Big-endian
+
+on:
+  pull_request:
+    branches: [ master ]
+    paths:
+    - '**.pyx'
+    - '**.pxd'
+    - 'src/borg/**/*.c'
+    - 'src/borg/**/*.h'
+    - 'src/borg/chunkers/**'
+    - 'src/borg/crypto/**'
+    - 'src/borg/repository.py'
+    - 'src/borg/repoobj.py'
+    - 'src/borg/archive.py'
+    - 'scripts/endian_interop_test.py'
+    - '.github/workflows/bigendian.yml'
+  schedule:
+    - cron: '43 5 * * 3'  # Wednesdays at 05:43 UTC
+  workflow_dispatch:      # Allow manual trigger
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  contents: read
+
+env:
+  PY_COLORS: "1"
+
+jobs:
+  s390x:
+    name: s390x (big-endian, emulated)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+
+    env:
+      # Debian trixie has python 3.13, and all our dependencies that ship binary
+      # wheels (blake3, backports-zstd, PyYAML, cffi) have s390x wheels for it -
+      # only the small C/Cython extensions are compiled (emulated) from source.
+      IMAGE: docker.io/s390x/debian:trixie-slim
+      CONTAINER: borg-s390x
+      # both sides of the interoperability test share this directory:
+      INTEROP: ${{ github.workspace }}/.interop
+      # a hung emulated test must fail instead of eating the job timeout:
+      PYTEST_TIMEOUT: "600"
+      # The endianness sensitive parts: the native code (chunkers, crypto,
+      # compression, hashindex, item) and the code that reads/writes the
+      # repository format. Extend this if it turns out to be too narrow.
+      PYTEST_TARGETS: >-
+        borg.testsuite.chunkers
+        borg.testsuite.crypto
+        borg.testsuite.compress_test
+        borg.testsuite.hashindex_test
+        borg.testsuite.item_test
+        borg.testsuite.repoobj_test
+        borg.testsuite.repository_test
+        borg.testsuite.archive_test
+        borg.testsuite.helpers.msgpack_test
+
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        # Just fetching one commit is not enough for setuptools-scm, so we fetch all.
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Set up Python
+      uses: actions/setup-python@v7
+      with:
+        # same python version as in the container, so both sides are comparable
+        python-version: '3.13'
+
+    # The wheels pip has to build from source inside the emulated container
+    # (msgpack, argon2-cffi-bindings, borghash, borgstore) are the expensive
+    # part of the container setup, so keep them from run to run.
+    - name: Cache pip-built wheels (s390x)
+      uses: actions/cache@v6
+      with:
+        path: .pip-cache-s390x
+        key: s390x-pip-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          s390x-pip-
+
+    - name: Install Linux packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pkg-config build-essential
+        sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev
+
+    - name: Build borg (native, little-endian)
+      run: |
+        set -euxo pipefail
+        # Note: a non-editable install on purpose - the s390x build below uses
+        # the same source tree and an editable install would put the extension
+        # modules of one architecture in there for the other one to pick up.
+        python -m venv "$RUNNER_TEMP/venv-native"
+        "$RUNNER_TEMP/venv-native/bin/pip" install --upgrade pip wheel
+        # Note: no pytest and no other development dependencies in this venv on
+        # purpose - this is what a normal borg installation looks like.
+        "$RUNNER_TEMP/venv-native/bin/pip" install .
+        "$RUNNER_TEMP/venv-native/bin/borg" --version
+        "$RUNNER_TEMP/venv-native/bin/python" -c 'import sys; assert sys.byteorder == "little", sys.byteorder'
+        # the container has no git, so tell setuptools-scm the version we got here:
+        echo "BORG_VERSION=$("$RUNNER_TEMP/venv-native/bin/borg" --version | cut -d' ' -f2)" >> $GITHUB_ENV
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: s390x
+
+    - name: Start the s390x container
+      run: |
+        set -euxo pipefail
+        mkdir -p .pip-cache-s390x
+        docker run -d --name "$CONTAINER" --platform linux/s390x \
+          -v "${{ github.workspace }}:/borg" -w /borg \
+          -e PIP_CACHE_DIR=/borg/.pip-cache-s390x \
+          -e SETUPTOOLS_SCM_PRETEND_VERSION_FOR_BORGBACKUP="$BORG_VERSION" \
+          -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
+          "$IMAGE" sleep infinity
+        # if this says anything but s390x, the emulation did not kick in:
+        test "$(docker exec "$CONTAINER" uname -m)" = "s390x"
+
+    - name: Build borg (s390x, big-endian)
+      run: |
+        docker exec -i "$CONTAINER" bash -s <<'EOF'
+        set -euxo pipefail
+        export DEBIAN_FRONTEND=noninteractive
+        apt-get update
+        apt-get install -y --no-install-recommends \
+          python3 python3-dev python3-venv \
+          build-essential pkg-config libssl-dev libacl1-dev liblz4-dev
+        python3 -c 'import sys; assert sys.byteorder == "big", sys.byteorder; print("byteorder:", sys.byteorder)'
+        python3 -m venv /venv
+        /venv/bin/pip install --upgrade pip wheel
+        /venv/bin/pip install pytest pytest-xdist pytest-benchmark pytest-timeout
+        /venv/bin/pip install /borg
+        /venv/bin/borg --version
+        chown -R "$HOST_UID:$HOST_GID" /borg/.pip-cache-s390x
+        EOF
+
+    - name: Run the endianness sensitive tests on s390x
+      run: |
+        docker exec -i \
+          -e PYTEST_TARGETS="$PYTEST_TARGETS" \
+          -e PYTEST_TIMEOUT="$PYTEST_TIMEOUT" \
+          -e PY_COLORS="$PY_COLORS" \
+          "$CONTAINER" bash -s <<'EOF'
+        set -euxo pipefail
+        # run against the installed borg, not against the source tree:
+        cd /tmp
+        /venv/bin/python -m pytest -v -n4 -rs --benchmark-skip --pyargs $PYTEST_TARGETS
+        EOF
+
+    - name: Write test data and a repository on s390x
+      run: |
+        set -euxo pipefail
+        "$RUNNER_TEMP/venv-native/bin/python" scripts/endian_interop_test.py testdata "$INTEROP"
+        docker exec -i -e INTEROP=/borg/.interop "$CONTAINER" bash -s <<'EOF'
+        set -euxo pipefail
+        BORG=/venv/bin/borg /venv/bin/python /borg/scripts/endian_interop_test.py write "$INTEROP" be
+        chown -R "$HOST_UID:$HOST_GID" "$INTEROP"
+        EOF
+
+    - name: Read the s390x repository on x86_64 (and write to it)
+      run: |
+        set -euxo pipefail
+        export BORG="$RUNNER_TEMP/venv-native/bin/borg"
+        PY="$RUNNER_TEMP/venv-native/bin/python"
+        $PY scripts/endian_interop_test.py verify "$INTEROP" be --as le
+        $PY scripts/endian_interop_test.py write "$INTEROP" le
+        $PY scripts/endian_interop_test.py compare "$INTEROP" be le
+
+    - name: Read the x86_64 archives on s390x
+      run: |
+        docker exec -i -e INTEROP=/borg/.interop "$CONTAINER" bash -s <<'EOF'
+        set -euxo pipefail
+        BORG=/venv/bin/borg /venv/bin/python /borg/scripts/endian_interop_test.py verify "$INTEROP" le --as be
+        chown -R "$HOST_UID:$HOST_GID" "$INTEROP"
+        EOF
+
+    - name: Stop the s390x container
+      if: ${{ always() }}
+      run: docker rm -f "$CONTAINER" || true
diff --git a/scripts/endian_interop_test.py b/scripts/endian_interop_test.py
new file mode 100644
index 0000000000..e9935f22c0
--- /dev/null
+++ b/scripts/endian_interop_test.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+"""
+Cross-architecture repository interoperability test.
+
+borg's on-disk / on-the-wire format is architecture independent, but almost all
+of our CI runs on little-endian machines only, so the big-endian code paths in
+the native code (e.g. the __builtin_bswap64() in the chunker kernels) are never
+executed.  This script drives a repository from two sides, so a repository that
+was written on one architecture can be verified on the other one:
+
+    # on machine/container A (e.g. big-endian):
+    endian_interop_test.py testdata WORKDIR       # deterministic test data
+    endian_interop_test.py write    WORKDIR be    # write archives "be-*"
+
+    # on machine/container B (e.g. little-endian), same WORKDIR:
+    endian_interop_test.py verify   WORKDIR be    # read what A wrote
+    endian_interop_test.py write    WORKDIR le    # write archives "le-*"
+    endian_interop_test.py compare  WORKDIR be le # same data, same chunks?
+
+    # on machine/container A again:
+    endian_interop_test.py verify   WORKDIR le    # read what B wrote
+
+"verify" runs "borg check --verify-data", extracts the other side's archives and
+compares the extracted files against the test data.  "compare" additionally
+requires that both sides cut the data into exactly the same chunks with exactly
+the same chunk IDs - if they did not, the archives would still be correct, but
+deduplication between machines of different endianness would silently not work.
+
+The environment (BORG_REPO, BORG_PASSPHRASE, cache and config dir) is set up by
+this script; the borg to use can be given via the BORG environment variable.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import random
+import shutil
+import subprocess
+import sys
+
+# fastcdc is borg's default chunker (and its kernel is one of the places with a
+# byte-order dependent code path).  Much smaller chunks than the default: the
+# test data then gets cut into thousands of chunks == thousands of chances to
+# notice a byte-order dependent cut point.
+#
+# The second archive uses different chunker parameters on purpose: with the same
+# parameters it would just deduplicate against the first one and its compression
+# would never be exercised.  So both sides also have to agree about the chunk
+# format of more than one compression method.
+ARCHIVE_SPECS = [
+    ("fastcdc-zstd", ["--chunker-params", "fastcdc,10,16,12,2", "--compression", "zstd,3"]),
+    ("fastcdc-lz4", ["--chunker-params", "fastcdc,12,18,14,2", "--compression", "lz4"]),
+]
+
+PASSPHRASE = "borg-endian-interop-test"
+
+# the test data lives in this subdirectory of the work directory
+DATA = "data"
+
+
+def borg(workdir, label, *args, cwd=None, capture=False):
+    """run a borg command against the test repository"""
+    env = os.environ.copy()
+    env["BORG_REPO"] = os.path.join(workdir, "repo")
+    env["BORG_PASSPHRASE"] = PASSPHRASE
+    # the KDF does not need to be strong for a throwaway test repository and
+    # argon2 is *slow* under emulation:
+    env["BORG_TESTONLY_WEAKEN_KDF"] = "1"
+    # keep the two sides (and the machine's own borg config) apart:
+    env["BORG_BASE_DIR"] = os.path.join(workdir, "home-%s" % label)
+    env["BORG_CACHE_DIR"] = os.path.join(workdir, "home-%s" % label, "cache")
+    env["BORG_CONFIG_DIR"] = os.path.join(workdir, "home-%s" % label, "config")
+    cmd = [os.environ.get("BORG", "borg")] + [str(a) for a in args]
+    print("+ %s" % " ".join(cmd), flush=True)
+    if capture:
+        return subprocess.run(cmd, env=env, cwd=cwd, check=True, stdout=subprocess.PIPE).stdout
+    subprocess.run(cmd, env=env, cwd=cwd, check=True)
+    return None
+
+
+def datadir(workdir):
+    return os.path.join(workdir, DATA)
+
+
+def cmd_testdata(args):
+    """create deterministic test data - content must not depend on the machine"""
+    path = datadir(args.workdir)
+    if os.path.exists(path):
+        shutil.rmtree(path)
+    os.makedirs(os.path.join(path, "nested", "dir"))
+    rnd = random.Random(20260814)
+
+    # incompressible, gets cut into many chunks:
+    with open(os.path.join(path, "random.bin"), "wb") as f:
+        f.write(rnd.randbytes(8 * 1024 * 1024))
+    # compressible, but not uniform - many different chunker cut points:
+    with open(os.path.join(path, "text.bin"), "wb") as f:
+        for i in range(64 * 1024):
+            f.write(b"line %d: %s\n" % (i, b"borg" * (i % 17)))
+    # all-zero data (the chunkers have a shortcut for this):
+    with open(os.path.join(path, "zeros.bin"), "wb") as f:
+        f.write(b"\0" * (2 * 1024 * 1024))
+    # the same data as random.bin, but shifted by a few bytes: the rolling hash
+    # has to find the same cut points again after the insertion.
+    with open(os.path.join(path, "random.bin"), "rb") as f:
+        data = f.read()
+    with open(os.path.join(path, "nested", "shifted.bin"), "wb") as f:
+        f.write(b"insertion" + data)
+    with open(os.path.join(path, "nested", "dir", "small.txt"), "wb") as f:
+        f.write(b"hello borg\n")
+    with open(os.path.join(path, "nested", "empty"), "wb"):
+        pass
+    with open(os.path.join(path, "nested", "\u00fcmlaut-\u65e5\u672c\u8a9e.txt"), "wb") as f:
+        f.write("non-ascii file name and content: \u00e4\u00f6\u00fc\n".encode())
+    os.symlink("dir/small.txt", os.path.join(path, "nested", "symlink"))
+    os.link(os.path.join(path, "nested", "dir", "small.txt"), os.path.join(path, "nested", "hardlink"))
+    print("test data created in %s" % path)
+
+
+def scan(path):
+    """map relative path -> file content hash / symlink target, for comparing 2 trees"""
+    result = {}
+    for dirpath, dirnames, filenames in os.walk(path):
+        dirnames.sort()
+        for name in sorted(dirnames + filenames):
+            full = os.path.join(dirpath, name)
+            rel = os.path.relpath(full, path)
+            if os.path.islink(full):
+                result[rel] = "symlink:%s" % os.readlink(full)
+            elif os.path.isdir(full):
+                result[rel] = "dir"
+            else:
+                digest = hashlib.sha256()
+                with open(full, "rb") as f:
+                    for block in iter(lambda: f.read(1024 * 1024), b""):
+                        digest.update(block)
+                result[rel] = "file:%s" % digest.hexdigest()
+    return result
+
+
+def dump_path(workdir, archive, label):
+    return os.path.join(workdir, "dumps", "%s.read-by-%s.json" % (archive, label))
+
+
+def dump_archive(workdir, label, archive):
+    """dump the archive metadata as read by *this* machine"""
+    os.makedirs(os.path.join(workdir, "dumps"), exist_ok=True)
+    borg(workdir, label, "debug", "dump-archive", archive, dump_path(workdir, archive, label))
+
+
+def chunk_list(dump_file):
+    """(path, size, chunks) of all items, everything that must not depend on the architecture"""
+    with open(dump_file) as f:
+        dump = json.load(f)
+    items = []
+    for item in dump["_items"]:
+        items.append((item["path"], item.get("size"), item.get("target"), item.get("chunks")))
+    return items
+
+
+def cmd_write(args):
+    workdir, label = args.workdir, args.label
+    if not os.path.exists(os.path.join(workdir, "repo")):
+        borg(workdir, label, "repo-create", "--encryption=aes256-ocb")
+    for suffix, options in ARCHIVE_SPECS:
+        archive = "%s-%s" % (label, suffix)
+        # --files-cache=disabled: really chunk the files, do not trust any cache.
+        # Archive the relative path "data" from within the work directory: the
+        # two sides usually see the work directory at different absolute paths
+        # (e.g. one of them inside a container), but the item paths in the
+        # archives must be comparable.
+        borg(workdir, label, "create", "--files-cache=disabled", *options, archive, DATA, cwd=workdir)
+        dump_archive(workdir, label, archive)
+    borg(workdir, label, "repo-list")
+
+
+def cmd_verify(args):
+    """check and extract the archives written by the *other* side"""
+    workdir, other, label = args.workdir, args.label, args.as_label
+    borg(workdir, label, "check", "--verify-data")
+    expected = scan(datadir(workdir))
+    assert expected, "no test data found in %s" % datadir(workdir)
+    for suffix, _ in ARCHIVE_SPECS:
+        archive = "%s-%s" % (other, suffix)
+        extract_to = os.path.join(workdir, "extract-%s-by-%s" % (archive, label))
+        if os.path.exists(extract_to):
+            shutil.rmtree(extract_to)
+        os.makedirs(extract_to)
+        borg(workdir, label, "extract", archive, cwd=extract_to)
+        got = scan(os.path.join(extract_to, DATA))
+        if got != expected:
+            for key in sorted(set(expected) | set(got)):
+                if expected.get(key) != got.get(key):
+                    print("MISMATCH %s: expected %r, got %r" % (key, expected.get(key), got.get(key)))
+            raise SystemExit("archive %s does not extract to the original test data!" % archive)
+        print("OK: %s extracted identically (%d entries)" % (archive, len(got)))
+        # the same archive, but read (and its metadata decoded) on this machine:
+        dump_archive(workdir, label, archive)
+        written_by_other = chunk_list(dump_path(workdir, archive, other))
+        read_by_us = chunk_list(dump_path(workdir, archive, label))
+        if written_by_other != read_by_us:
+            raise SystemExit("archive %s does not decode identically on both architectures!" % archive)
+        print("OK: %s metadata (incl. chunk ids) decodes identically on both architectures" % archive)
+
+
+def cmd_compare(args):
+    """both sides archived the same data: they must have produced the same chunks"""
+    workdir, a, b = args.workdir, args.label, args.other_label
+    for suffix, _ in ARCHIVE_SPECS:
+        archive_a, archive_b = "%s-%s" % (a, suffix), "%s-%s" % (b, suffix)
+        items_a = chunk_list(dump_path(workdir, archive_a, a))
+        items_b = chunk_list(dump_path(workdir, archive_b, b))
+        if items_a != items_b:
+            for item_a, item_b in zip(items_a, items_b):
+                if item_a != item_b:
+                    print("MISMATCH %s:\n  %s: %r\n  %s: %r" % (item_a[0], a, item_a[1:], b, item_b[1:]))
+            raise SystemExit(
+                "%s and %s chunked the same data differently - "
+                "deduplication between these architectures would not work!" % (archive_a, archive_b)
+            )
+        chunks = sum(len(item[3] or []) for item in items_a)
+        print("OK: %s and %s have identical items and chunk ids (%d chunks)" % (archive_a, archive_b, chunks))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    sub = commands.add_parser("testdata", help="create the deterministic test data")
+    sub.add_argument("workdir")
+    sub.set_defaults(func=cmd_testdata)
+
+    sub = commands.add_parser("write", help="create the repository and this side's archives")
+    sub.add_argument("workdir")
+    sub.add_argument("label", help="label of this side, e.g. 'be'")
+    sub.set_defaults(func=cmd_write)
+
+    sub = commands.add_parser("verify", help="check/extract the archives the other side wrote")
+    sub.add_argument("workdir")
+    sub.add_argument("label", help="label of the *other* side, e.g. 'be'")
+    sub.add_argument("--as", dest="as_label", required=True, help="label of this side, e.g. 'le'")
+    sub.set_defaults(func=cmd_verify)
+
+    sub = commands.add_parser("compare", help="compare the chunk ids both sides produced")
+    sub.add_argument("workdir")
+    sub.add_argument("label")
+    sub.add_argument("other_label")
+    sub.set_defaults(func=cmd_compare)
+
+    args = parser.parse_args()
+    os.makedirs(args.workdir, exist_ok=True)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

From 49bed0c4b545f181e1393d2deb53d8e816c35614 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 15 Aug 2026 07:56:28 +0200
Subject: [PATCH 133/144] CI: give the black workflow read-only token
 permissions

It was the only workflow without a permissions declaration, so it got
whatever the repository default is. It only checks out and runs black.
---
 .github/workflows/black.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index 94c22ea57d..146c3beba3 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -19,6 +19,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
+permissions:
+  contents: read
+
 jobs:
   lint:
     runs-on: ubuntu-24.04

From abd9c3d88cb92f00b735892be74d058b108131f3 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 15 Aug 2026 08:52:14 +0200
Subject: [PATCH 134/144] CI: release automation for PyPI and GitHub releases

Pushing a release tag so far only built the standalone binaries and uploaded
them as workflow artifacts - everything else was manual: build and sign the
sdist, upload it to PyPI, fetch the binary artifacts and create the GitHub
release with them.

Add a release job (.github/workflows/release.yml), called by ci.yml for tags
after the tests and the binary builds succeeded - it runs inside the same
workflow run, because that is where the binary artifacts are. It builds the
sdist, verifies that the sdist is complete by installing it into a clean venv
and running borg, attests its provenance and drafts the GitHub release with
the sdist and all standalone binaries attached.

A pypi job then uploads the sdist to PyPI via trusted publishing, so no API
token has to be stored anywhere. It lives in ci.yml because PyPI trusted
publishing does not work from a reusable workflow. It uses the "pypi"
environment: configuring required reviewers for it makes the irreversible
upload wait for an approval.

The Windows binaries now use the same naming and layout as the binaries of
the other platforms (borg-windows-x86_64-gh.exe and .tgz, with the
single-directory variant and a provenance attestation), so they become
release assets, too.

The GitHub release is created as a draft: the release notes want a human, the
detached GPG signature of the sdist can only be made locally, and the binaries
should be tried out before the release becomes visible.
---
 .github/workflows/ci.yml      |  95 ++++++++++++++++++-
 .github/workflows/release.yml | 171 ++++++++++++++++++++++++++++++++++
 docs/development.rst          |  37 ++++----
 3 files changed, 280 insertions(+), 23 deletions(-)
 create mode 100644 .github/workflows/release.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 334ef1ca13..edbec097ce 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -685,6 +685,11 @@ jobs:
     timeout-minutes: 90
     needs: [lint]
 
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+
     env:
       MSYS2_ARG_CONV_EXCL: "*"
       MSYS2_ENV_CONV_EXCL: "*"
@@ -738,10 +743,36 @@ jobs:
           # build sdist and wheel in dist/...
           python -m build
 
-      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+      # Same layout and naming as the binaries of the other platforms, so that
+      # the release job picks this up as a release asset, too. The single-file
+      # binary keeps its .exe extension - Windows needs it to run the file.
+      - name: Prepare binaries (borg-windows-x86_64-gh)
+        run: |
+          pushd dist/binary
+          echo "single-file binary"
+          ./borg.exe -V
+          echo "single-directory binary"
+          ./borg-dir/borg.exe -V
+          tar czf borg.tgz borg-dir
+          popd
+          mkdir -p artifacts
+          cp dist/binary/borg.exe artifacts/borg-windows-x86_64-gh.exe
+          cp dist/binary/borg.tgz artifacts/borg-windows-x86_64-gh.tgz
+          echo "binary files"
+          ls -l artifacts/
+
+      - name: Attest binaries provenance (borg-windows-x86_64-gh)
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8  # v4.2.2
         with:
-          name: borg-windows
-          path: dist/binary/borg.exe
+          subject-path: 'artifacts/*'
+
+      - name: Upload binaries (borg-windows-x86_64-gh)
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: borg-windows-x86_64-gh
+          path: artifacts/*
+          if-no-files-found: error
 
       - name: Run tests
         run: |
@@ -774,3 +805,61 @@ jobs:
           report_type: coverage
           env_vars: OS,python
           files: coverage.xml
+
+  release:
+    # Build the sdist and draft the GitHub release with the binaries the jobs
+    # above built for this tag, see .github/workflows/release.yml.
+    #
+    # vm_tests is continue-on-error (a flaky BSD VM must not stop a release), so
+    # this waits for it, but only requires native_tests to have succeeded. A
+    # binary that did not get built is warned about while drafting the release.
+    if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/') && needs.native_tests.result == 'success' }}
+    needs: [native_tests, vm_tests]
+
+    permissions:
+      contents: write
+      id-token: write
+      attestations: write
+
+    uses: ./.github/workflows/release.yml
+
+  pypi:
+    # Upload the sdist the release job built to PyPI.
+    #
+    # This job can not live in release.yml with the rest of the release code:
+    # PyPI trusted publishing does not work from a reusable workflow, see
+    # https://docs.pypi.org/trusted-publishers/troubleshooting/
+    #
+    # One-time setup, so that no API token has to be stored anywhere:
+    #  - on pypi.org, add a trusted publisher to the "borgbackup" project:
+    #    owner "borgbackup", repository "borg", workflow "ci.yml",
+    #    environment "pypi".
+    #  - create the "pypi" environment in the repository settings. Configuring
+    #    required reviewers for it makes this (irreversible) upload wait for an
+    #    approval, which is the last chance to stop a release.
+    if: ${{ startsWith(github.ref, 'refs/tags/') && needs.release.result == 'success' }}
+    needs: [release]
+
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+
+    environment:
+      name: pypi
+      url: https://pypi.org/project/borgbackup/
+
+    permissions:
+      contents: read
+      id-token: write  # trusted publishing
+
+    steps:
+    - name: Get the sdist built by the release job
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+      with:
+        name: sdist
+        path: dist
+
+    - name: What we are about to upload
+      run: ls -l dist/
+
+    - name: Upload to PyPI
+      uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33  # v1.14.2
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000000..4a9a29ebb8
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,171 @@
+# Release automation: build the source distribution, publish it to PyPI and
+# create a GitHub release with the standalone binaries.
+#
+# This is called by ci.yml when a tag is pushed, and it deliberately runs as
+# part of that same workflow run: that is where the binaries for the tag are
+# built, and artifacts can only be downloaded within the run that created them.
+#
+# The GitHub release is created as a *draft* on purpose:
+#  - the release notes want a human,
+#  - the detached GPG signature of the sdist can only be made locally
+#    (scripts/sdist-sign), so it has to be added by hand,
+#  - and the binaries should be tried out before the release becomes visible.
+# Publishing the draft is a single click in the GitHub UI.
+#
+# The upload to PyPI is *not* done here, but by the "pypi" job in ci.yml: PyPI
+# trusted publishing can not be used from a reusable workflow, see
+# https://docs.pypi.org/trusted-publishers/troubleshooting/ - so that job has to
+# live in a workflow that is triggered by an event. This job hands the sdist
+# over to it as a workflow artifact.
+
+name: Release
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+env:
+  # The standalone binaries expected for a release: for every platform the
+  # single-file binary and, as a .tgz, the single-directory variant.  See the
+  # "binary" entries of the native_tests matrix, the "artifact_prefix" entries
+  # of the vm_tests matrix and the windows_tests job in ci.yml - keep in sync.
+  EXPECTED_ASSETS: >-
+    borg-linux-glibc239-x86_64-gh
+    borg-linux-glibc239-x86_64-gh.tgz
+    borg-linux-glibc239-arm64-gh
+    borg-linux-glibc239-arm64-gh.tgz
+    borg-macos-15-arm64-gh
+    borg-macos-15-arm64-gh.tgz
+    borg-macos-15-x86_64-gh
+    borg-macos-15-x86_64-gh.tgz
+    borg-freebsd-15-x86_64-gh
+    borg-freebsd-15-x86_64-gh.tgz
+    borg-windows-x86_64-gh.exe
+    borg-windows-x86_64-gh.tgz
+
+jobs:
+  github_release:
+    name: Draft the GitHub release
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+
+    permissions:
+      contents: write      # to create the release
+      id-token: write      # to attest the sdist
+      attestations: write
+
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      with:
+        # Just fetching one commit is not enough for setuptools-scm, so we fetch all.
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Set up Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+      with:
+        python-version: '3.13'
+
+    - name: Install Linux packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pkg-config build-essential
+        sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev
+
+    - name: Build the sdist
+      run: |
+        set -euxo pipefail
+        python -m pip install --upgrade pip build twine
+        # only a sdist: we do not publish wheels, they would be platform specific.
+        python -m build --sdist
+        twine check dist/*
+        ls -l dist/
+
+    - name: Check that the sdist is complete and installable
+      # A release that cannot be installed from PyPI is the worst kind of
+      # release, and nothing else in CI ever installs borg from a sdist or
+      # without the development requirements.
+      run: |
+        set -euxo pipefail
+        python -m venv "$RUNNER_TEMP/venv-sdist"
+        "$RUNNER_TEMP/venv-sdist/bin/pip" install --upgrade pip
+        "$RUNNER_TEMP/venv-sdist/bin/pip" install dist/borgbackup-*.tar.gz
+        "$RUNNER_TEMP/venv-sdist/bin/borg" --version
+        "$RUNNER_TEMP/venv-sdist/bin/borg" --help > /dev/null
+
+    - name: Attest the sdist provenance
+      uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8  # v4.2.2
+      with:
+        subject-path: 'dist/*.tar.gz'
+
+    - name: Download the binaries built for this tag
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+      with:
+        path: assets
+        # the binary artifacts are all named like the binaries they contain
+        pattern: 'borg-*-gh'
+        merge-multiple: true
+
+    - name: Check that no binary is missing
+      run: |
+        set -uo pipefail
+        ls -l assets/ || true
+        missing=""
+        for asset in $EXPECTED_ASSETS; do
+          test -f "assets/$asset" || missing="$missing $asset"
+        done
+        if [ -n "$missing" ]; then
+          # not an error: vm_tests is continue-on-error, so e.g. a flaky FreeBSD
+          # VM should not stop the release - but do not lose it silently either.
+          echo "::warning::binaries missing from this release:$missing"
+        fi
+
+    - name: Create the draft release
+      env:
+        GH_TOKEN: ${{ github.token }}
+        TAG: ${{ github.ref_name }}
+      run: |
+        set -euxo pipefail
+        # 2.0.0b23 and friends are pre-releases, 2.1.0 is not.
+        prerelease=""
+        case "$TAG" in *a*|*b*|*rc*) prerelease="--prerelease" ;; esac
+        cat > release-notes.md <<EOF
+        See the [changelog](https://github.com/borgbackup/borg/blob/$TAG/CHANGES.rst) for what changed in this release.
+
+        ### Installation
+
+        \`pip install borgbackup==$TAG\`, or use one of the standalone binaries below - they
+        contain everything they need and just have to be made executable.
+
+        For each platform there is a single-file binary (\`borg-*-gh\`, \`.exe\` on Windows)
+        and, as a \`.tgz\`, the same thing as a directory (\`borg-dir/borg.exe\`), which
+        starts up faster.
+
+        The macOS binaries are built **with** FUSE support: \`borg mount\` needs a macFUSE
+        version that is compatible with the one they were built against.
+
+        All release assets have a [build provenance attestation](https://github.com/borgbackup/borg/attestations),
+        verifiable with \`gh attestation verify --owner borgbackup <file>\`.
+        EOF
+        mkdir -p assets  # there may not have been any artifact to download
+        if gh release view "$TAG" > /dev/null 2>&1; then
+          # a re-run of this job: keep the (possibly already edited) release and
+          # just replace its assets.
+          gh release upload "$TAG" --clobber dist/*.tar.gz $(find assets -type f | sort)
+        else
+          gh release create "$TAG" \
+            --draft $prerelease \
+            --title "borg $TAG" \
+            --notes-file release-notes.md \
+            dist/*.tar.gz $(find assets -type f | sort)
+        fi
+        gh release view "$TAG" --json isDraft,isPrerelease,assets
+
+    - name: Keep the sdist for the PyPI upload
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+      with:
+        name: sdist
+        path: dist/*.tar.gz
+        if-no-files-found: error
diff --git a/docs/development.rst b/docs/development.rst
index 4a07ff70f9..34910e56da 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -531,22 +531,29 @@ Checklist:
 - Optional: run tox and/or binary builds on all supported platforms via vagrant,
   check for test failures. This is now optional as we do platform testing and
   binary building on GitHub.
-- Create sdist, sign it, upload release to (test) PyPi:
+- When GitHub CI looks good on the release PR, merge it and push the release tag.
 
-  ::
+  Pushing the tag makes CI build the standalone binaries and then release:
+  the ``release`` job (``.github/workflows/release.yml``) builds the sdist,
+  checks that it can be installed, attests its provenance and drafts the GitHub
+  release with the sdist and all standalone binaries attached. The ``pypi`` job
+  then uploads the sdist to PyPI.
+
+  If the ``pypi`` environment has required reviewers configured, the upload to
+  PyPI waits for an approval - that is the last chance to stop it. Also watch
+  out for a warning while the release is drafted: it means that a binary is
+  missing because its build did not succeed.
+- Sign the sdist and add the signature to the drafted GitHub release::
 
     scripts/sdist-sign X.Y.Z
-    scripts/upload-pypi X.Y.Z test
-    scripts/upload-pypi X.Y.Z
 
   Note: the signature is not uploaded to PyPi any more, but we upload it to
   github releases.
-- When GitHub CI looks good on the release PR, merge it and then check "Actions":
-  GitHub will create binary assets after the release PR is merged within the
-  CI testing of the merge. Check the "Upload binaries" step on Ubuntu (AMD/Intel
-  and ARM64) and macOS (Intel and ARM64), fetch the ZIPs with the binaries.
-- Unpack the ZIPs and test the binaries, upload the binaries to the GitHub
-  release page (borg-OS-SPEC-ARCH-gh and borg-OS-SPEC-ARCH-gh.tgz).
+- Download the binaries from the drafted release and test them. For macOS
+  binaries **with** FUSE support, document the macFUSE version in the release
+  notes: macFUSE uses a kernel extension that needs to be compatible with the
+  code contained in the binary.
+- Review the release notes and publish the drafted GitHub release.
 
 - Close the release milestone on GitHub.
 - `Update borgbackup.org
@@ -557,13 +564,3 @@ Checklist:
   - Mailing list.
   - Mastodon / BlueSky / X (aka Twitter).
   - IRC channel (change ``/topic``).
-
-- Create a GitHub release, include:
-
-  - pypi dist package and signature
-  - Standalone binaries (see above for how to create them).
-
-    - For macOS binaries **with** FUSE support, document the macFUSE version
-      in the README of the binaries. macFUSE uses a kernel extension that needs
-      to be compatible with the code contained in the binary.
-  - A link to ``CHANGES.rst``.

From d0baab45d1f3236f7be205888bc3a3cac0dec909 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 15 Aug 2026 13:20:48 +0200
Subject: [PATCH 135/144] windows: restore timestamps via SetFileTime, fixes
 #7269

os.utime is a bad fit for restoring timestamps on Windows:

- it can not set the birthtime (creation time), so the birthtime was
  archived (Python >= 3.12 has st_birthtime_ns there), but never restored.
- it does not support follow_symlinks=False there, so the timestamps of an
  extracted symlink were set on the symlink's target.
- it does not support file descriptors there, so we had to use the path
  even though we had the file open already.

Add platform.set_times() which does all that: the POSIX implementation is
the os.utime code moved from Archive.restore_attrs (incl. the utimes trick
to set the birthtime on the BSDs and Darwin), the win32 implementation uses
CreateFileW / SetFileTime.

Also, failing to set the timestamps is not silently ignored on win32 anymore,
but warns and sets the warning exit code.
---
 docs/changes.rst                              |  6 ++
 src/borg/archive.py                           | 53 +++++---------
 src/borg/platform/__init__.py                 |  2 +
 src/borg/platform/base.py                     | 26 +++++++
 src/borg/platform/windows.pyx                 | 69 ++++++++++++++++++
 .../testsuite/archiver/extract_cmd_test.py    | 22 ++++++
 src/borg/testsuite/platform/windows_test.py   | 72 ++++++++++++++++++-
 7 files changed, 212 insertions(+), 38 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 5ccd15cc17..f88a6a6513 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -259,6 +259,12 @@ Fixes:
   did not notice content changes of files that kept their size and inode number.
   The default is ``mtime,size,inode`` there now and an explicitly given ctime based
   mode warns and uses the corresponding mtime based mode.
+- extract: restore the timestamps using SetFileTime on Windows, #7269.
+
+  os.utime can not set the birthtime (creation time) there, nor can it work on a file
+  descriptor or on a symlink itself. Thus, the birthtime was not restored at all and the
+  timestamps of a symlink were set on the symlink's target. Also, failing to set the
+  timestamps is not silently ignored anymore, but gives a warning.
 - re-add XXH64 to read borg 1.x integrity data, #9935
 - list: add {blake3} format key, #9984
 - support date: archive patterns for --from-borg1, #9949
diff --git a/src/borg/archive.py b/src/borg/archive.py
index db7de5cd03..4303ee6798 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -50,7 +50,7 @@
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem, ItemDiff
 from . import platform
-from .platform import acl_get, acl_set, set_flags, get_flags, swidth
+from .platform import acl_get, acl_set, set_flags, get_flags, set_times, swidth
 from .repository import Repository, NoManifestError
 from .repoobj import RepoObj
 
@@ -1096,44 +1096,23 @@ def restore_attrs(self, path, item, symlink=False, fd=None):
                 warning = xattr.set_all(fd or path, item.xattrs, follow_symlinks=False)
                 if warning:
                     set_ec(EXIT_WARNING)
-            # set timestamps rather late
-            mtime = item.mtime
-            atime = item.atime if "atime" in item else mtime
-            if "birthtime" in item:
-                birthtime = item.birthtime
-                try:
-                    # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms.
-                    # See utimes(2) on either of the BSDs for details.
-                    if fd:
-                        os.utime(fd, None, ns=(atime, birthtime))
-                    else:
-                        os.utime(path, None, ns=(atime, birthtime), follow_symlinks=False)
-                except OSError:
-                    # some systems don't support calling utime on a symlink
-                    pass
-            try:
-                if fd:
-                    os.utime(fd, None, ns=(atime, mtime))
-                else:
-                    os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
-            except OSError:
-                # some systems don't support calling utime on a symlink
-                pass
-            # bsdflags include the immutable flag and need to be set last:
-            if not self.noflags and "bsdflags" in item:
-                try:
-                    set_flags(path, item.bsdflags, fd=fd)
-                except OSError:
-                    pass
-        else:  # win32
-            # set timestamps rather late
-            mtime = item.mtime
-            atime = item.atime if "atime" in item else mtime
+        # set timestamps rather late
+        mtime = item.mtime
+        atime = item.atime if "atime" in item else mtime
+        birthtime = item.get("birthtime")
+        try:
+            set_times(path, atime_ns=atime, mtime_ns=mtime, birthtime_ns=birthtime, fd=fd, follow_symlinks=False)
+        except OSError as e:
+            # some POSIX systems don't support setting the timestamps of a symlink.
+            if is_win32:
+                # win32 can set the timestamps of a symlink itself, so this is a real problem.
+                logger.warning("%s: when setting timestamps: %s", remove_surrogates(item.path), e)
+                set_ec(EXIT_WARNING)
+        # bsdflags include the immutable flag and need to be set last:
+        if not is_win32 and not self.noflags and "bsdflags" in item:
             try:
-                # note: no fd support on win32
-                os.utime(path, None, ns=(atime, mtime))
+                set_flags(path, item.bsdflags, fd=fd)
             except OSError:
-                # some systems don't support calling utime on a symlink
                 pass
 
     def set_meta(self, key, value):
diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py
index 4ff1b77a07..4495a3444a 100644
--- a/src/borg/platform/__init__.py
+++ b/src/borg/platform/__init__.py
@@ -12,6 +12,7 @@
 from .base import SaveFile, sync_dir, fdatasync, safe_fadvise
 from .base import get_process_id, fqdn, hostname, hostid, swidth
 from .base import acl_text_to_xattr  # overridden below for platforms supporting it
+from .base import set_times  # overridden below for win32
 
 # work around pyinstaller "forgetting" to include the xattr module
 from . import xattr  # noqa: F401
@@ -84,6 +85,7 @@
     from .base import acl_get, acl_set
     from .base import set_flags, get_flags
     from .windows import SyncFile
+    from .windows import set_times  # type: ignore[no-redef]
     from .windows import process_alive, local_pid_alive
     from .windows import getosusername
     from . import windows_ug as platform_ug
diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py
index 21e8bf9081..38152f82d1 100644
--- a/src/borg/platform/base.py
+++ b/src/borg/platform/base.py
@@ -112,6 +112,32 @@ def get_flags(path, st, fd=None):
     return getattr(st, "st_flags", 0)
 
 
+def set_times(path, *, atime_ns, mtime_ns, birthtime_ns=None, fd=None, follow_symlinks=True):
+    """
+    Set the timestamps of *path* (or of the open file descriptor *fd*, if given).
+
+    *birthtime_ns* is only honoured on platforms that can set the birthtime (creation time),
+    it is silently ignored on the other ones.
+
+    Raises OSError if the timestamps could not be set.
+    """
+    if birthtime_ns is not None:
+        try:
+            # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms.
+            # See utimes(2) on either of the BSDs for details.
+            if fd is not None:
+                os.utime(fd, None, ns=(atime_ns, birthtime_ns))
+            else:
+                os.utime(path, None, ns=(atime_ns, birthtime_ns), follow_symlinks=follow_symlinks)
+        except OSError:
+            # some systems don't support calling utime on a symlink
+            pass
+    if fd is not None:
+        os.utime(fd, None, ns=(atime_ns, mtime_ns))
+    else:
+        os.utime(path, None, ns=(atime_ns, mtime_ns), follow_symlinks=follow_symlinks)
+
+
 def sync_dir(path):
     if is_win32:
         # Opening directories is not supported on Windows.
diff --git a/src/borg/platform/windows.pyx b/src/borg/platform/windows.pyx
index c7a192a385..075b181346 100644
--- a/src/borg/platform/windows.pyx
+++ b/src/borg/platform/windows.pyx
@@ -22,12 +22,24 @@ cdef extern from 'windows.h':
 # Win32 API constants for CreateFileW
 GENERIC_READ = 0x80000000
 GENERIC_WRITE = 0x40000000
+FILE_WRITE_ATTRIBUTES = 0x00000100
 FILE_SHARE_READ = 0x00000001
+FILE_SHARE_WRITE = 0x00000002
+FILE_SHARE_DELETE = 0x00000004
 CREATE_NEW = 1
+OPEN_EXISTING = 3
 FILE_ATTRIBUTE_NORMAL = 0x80
 FILE_FLAG_WRITE_THROUGH = 0x80000000
+FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
 ERROR_FILE_EXISTS = 80
 
+# a FILETIME counts 100ns intervals since 1601-01-01, our timestamps count ns since 1970-01-01.
+FILETIME_EPOCH_OFFSET_NS = 11644473600 * 1000000000
+# SetFileTime gives a special meaning to 0 ("do not change") and to all bits set
+# ("do not update this timestamp for this handle any more"), so avoid these values.
+FILETIME_MIN, FILETIME_MAX = 1, 0xFFFFFFFFFFFFFFFE
+
 _kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
 _CreateFileW = _kernel32.CreateFileW
 _CreateFileW.restype = ctypes.wintypes.HANDLE
@@ -41,6 +53,14 @@ _CreateFileW.argtypes = [
     ctypes.wintypes.HANDLE,
 ]
 _CloseHandle = _kernel32.CloseHandle
+_SetFileTime = _kernel32.SetFileTime
+_SetFileTime.restype = ctypes.wintypes.BOOL
+_SetFileTime.argtypes = [
+    ctypes.wintypes.HANDLE,
+    ctypes.POINTER(ctypes.wintypes.FILETIME),
+    ctypes.POINTER(ctypes.wintypes.FILETIME),
+    ctypes.POINTER(ctypes.wintypes.FILETIME),
+]
 INVALID_HANDLE_VALUE = ctypes.wintypes.HANDLE(-1).value
 
 
@@ -103,6 +123,55 @@ class SyncFile(BaseSyncFile):
         os.fsync(self.fd)
 
 
+def _ns_to_filetime(ns):
+    """Convert a timestamp in ns since 1970-01-01 to a Win32 FILETIME."""
+    ft = (ns + FILETIME_EPOCH_OFFSET_NS) // 100
+    # clamp to what a FILETIME can express (borg timestamps have a much wider range).
+    ft = min(max(ft, FILETIME_MIN), FILETIME_MAX)
+    return ctypes.wintypes.FILETIME(ft & 0xFFFFFFFF, ft >> 32)
+
+
+def set_times(path, *, atime_ns, mtime_ns, birthtime_ns=None, fd=None, follow_symlinks=True):
+    """
+    Set the timestamps of *path* (or of the open file descriptor *fd*, if given).
+
+    Uses SetFileTime rather than os.utime, because os.utime on Windows can neither work on
+    a file descriptor nor on a symlink itself, nor can it set the birthtime (creation time).
+
+    Raises OSError if the timestamps could not be set.
+    """
+    if fd is not None:
+        handle, close_handle = msvcrt.get_osfhandle(fd), False
+    else:
+        # FILE_FLAG_BACKUP_SEMANTICS is required to get a handle for a directory.
+        flags = FILE_FLAG_BACKUP_SEMANTICS
+        if not follow_symlinks:
+            flags |= FILE_FLAG_OPEN_REPARSE_POINT
+        handle = _CreateFileW(
+            str(path),
+            FILE_WRITE_ATTRIBUTES,
+            FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+            None,
+            OPEN_EXISTING,
+            flags,
+            None,
+        )
+        if handle == INVALID_HANDLE_VALUE:
+            raise ctypes.WinError(ctypes.get_last_error())
+        close_handle = True
+    try:
+        atime = _ns_to_filetime(atime_ns)
+        mtime = _ns_to_filetime(mtime_ns)
+        birthtime = _ns_to_filetime(birthtime_ns) if birthtime_ns is not None else None
+        if not _SetFileTime(
+            handle, ctypes.byref(birthtime) if birthtime is not None else None, ctypes.byref(atime), ctypes.byref(mtime)
+        ):
+            raise ctypes.WinError(ctypes.get_last_error())
+    finally:
+        if close_handle:
+            _CloseHandle(handle)
+
+
 def getosusername():
     """Return the OS username."""
     return os.getlogin()
diff --git a/src/borg/testsuite/archiver/extract_cmd_test.py b/src/borg/testsuite/archiver/extract_cmd_test.py
index d2a3e302c9..0e1668d426 100644
--- a/src/borg/testsuite/archiver/extract_cmd_test.py
+++ b/src/borg/testsuite/archiver/extract_cmd_test.py
@@ -326,6 +326,28 @@ def test_birthtime(archivers, request):
     assert same_ts_ns(sto.st_mtime_ns, mtime * 10**9)
 
 
+@pytest.mark.skipif(not is_win32, reason="Windows-only test")
+def test_timestamps_win32(archivers, request):
+    """windows: extract restores atime, mtime and (with Python >= 3.12) birthtime, see #7269"""
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file", contents=b"stuff")
+    # os.utime can not set the birthtime on Windows, so use our own platform code for the setup.
+    # a FILETIME has a resolution of 100ns, thus only use timestamps that are a multiple of 100ns.
+    atime_ns, mtime_ns, birthtime_ns = 1500000000000000100, 1400000000000000200, 1300000000000000300
+    platform.set_times("input/file", atime_ns=atime_ns, mtime_ns=mtime_ns, birthtime_ns=birthtime_ns)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "--atime", "test", "input")
+    sti = os.stat("input/file")
+    with changedir("output"):
+        cmd(archiver, "extract", "test")
+    sto = os.stat("output/input/file")
+    assert sto.st_mtime_ns == mtime_ns
+    # reading the input file for the backup might have updated its atime, so compare with the input file.
+    assert sto.st_atime_ns == sti.st_atime_ns
+    if hasattr(sti, "st_birthtime_ns"):  # Python >= 3.12
+        assert sto.st_birthtime_ns == birthtime_ns
+
+
 def test_sparse_file(archivers, request):
     archiver = request.getfixturevalue(archivers)
 
diff --git a/src/borg/testsuite/platform/windows_test.py b/src/borg/testsuite/platform/windows_test.py
index eeb6a4d0e1..4f61919689 100644
--- a/src/borg/testsuite/platform/windows_test.py
+++ b/src/borg/testsuite/platform/windows_test.py
@@ -1,13 +1,19 @@
+import os
 import tempfile
 
 import pytest
 
 from .platform_test import skipif_not_win32
-from ...platform import SyncFile
+from .. import are_symlinks_supported
+from ...platform import SyncFile, set_times
 
 # Set module-level skips
 pytestmark = [skipif_not_win32]
 
+# timestamps used by the set_times tests. a FILETIME has a resolution of 100ns,
+# so only use values that are a multiple of 100ns to be able to compare exactly.
+ATIME_NS, MTIME_NS, BIRTHTIME_NS = 1500000000000000100, 1400000000000000200, 1300000000000000300
+
 
 def test_syncfile_basic(tmp_path):
     """Integration: SyncFile creates file and writes data correctly."""
@@ -70,3 +76,67 @@ def mock_create(*args):
     assert len(calls) == 1
     flags_attrs = calls[0][5]  # 6th arg: dwFlagsAndAttributes
     assert flags_attrs & windows.FILE_FLAG_WRITE_THROUGH
+
+
+def assert_times(st, *, atime_ns=ATIME_NS, mtime_ns=MTIME_NS, birthtime_ns=BIRTHTIME_NS):
+    assert st.st_atime_ns == atime_ns
+    assert st.st_mtime_ns == mtime_ns
+    if hasattr(st, "st_birthtime_ns"):  # Python >= 3.12 on Windows
+        assert st.st_birthtime_ns == birthtime_ns
+
+
+def test_set_times_file(tmp_path):
+    """set_times sets atime, mtime and birthtime of a file given by path."""
+    path = tmp_path / "file"
+    path.write_bytes(b"data")
+    set_times(str(path), atime_ns=ATIME_NS, mtime_ns=MTIME_NS, birthtime_ns=BIRTHTIME_NS)
+    assert_times(os.stat(path))
+
+
+def test_set_times_fd(tmp_path):
+    """set_times works on an open file descriptor, like borg extract uses it."""
+    path = tmp_path / "file"
+    with open(path, "wb") as f:
+        f.write(b"data")
+        f.flush()
+        set_times(str(path), atime_ns=ATIME_NS, mtime_ns=MTIME_NS, birthtime_ns=BIRTHTIME_NS, fd=f.fileno())
+    # the timestamps must survive closing the file we have written to.
+    assert_times(os.stat(path))
+
+
+def test_set_times_directory(tmp_path):
+    """set_times works on a directory (needs FILE_FLAG_BACKUP_SEMANTICS)."""
+    path = tmp_path / "dir"
+    path.mkdir()
+    set_times(str(path), atime_ns=ATIME_NS, mtime_ns=MTIME_NS, birthtime_ns=BIRTHTIME_NS)
+    assert_times(os.stat(path))
+
+
+def test_set_times_without_birthtime(tmp_path):
+    """set_times leaves the birthtime alone if we do not give one."""
+    path = tmp_path / "file"
+    path.write_bytes(b"data")
+    birthtime_ns = getattr(os.stat(path), "st_birthtime_ns", None)
+    set_times(str(path), atime_ns=ATIME_NS, mtime_ns=MTIME_NS)
+    assert_times(os.stat(path), birthtime_ns=birthtime_ns)
+
+
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_set_times_symlink_not_followed(tmp_path):
+    """set_times(follow_symlinks=False) works on the symlink itself, not on its target."""
+    target = tmp_path / "target"
+    target.write_bytes(b"data")
+    target_times = os.stat(target)
+    link = tmp_path / "link"
+    os.symlink(str(target), str(link))
+    set_times(str(link), atime_ns=ATIME_NS, mtime_ns=MTIME_NS, birthtime_ns=BIRTHTIME_NS, follow_symlinks=False)
+    assert_times(os.stat(link, follow_symlinks=False))
+    st_target = os.stat(target)
+    assert st_target.st_mtime_ns == target_times.st_mtime_ns
+    assert st_target.st_atime_ns == target_times.st_atime_ns
+
+
+def test_set_times_nonexistent(tmp_path):
+    """set_times raises OSError if there is no such file."""
+    with pytest.raises(OSError):
+        set_times(str(tmp_path / "nonexistent"), atime_ns=ATIME_NS, mtime_ns=MTIME_NS)

From 64c79e113fbdbc96f8ae1fa0fa66235399d43076 Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Sat, 15 Aug 2026 18:44:24 +0530
Subject: [PATCH 136/144] check --repair: report index-referenced missing packs
 as errors, not a corrupt index

---
 src/borg/repository.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/borg/repository.py b/src/borg/repository.py
index 354f6943aa..a007f7561f 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -1195,8 +1195,6 @@ def recorded_ts(info):
             # run: sig_int breaks the loop early, so "no pack errors" must be paired with "all packs
             # scanned" (pack_files == len(pack_infos)) to not rebuild from unverified packs.
             if index_errors and pack_errors == 0 and not sig_int and pack_files == len(pack_infos):
-                from .cache import build_chunkindex_from_repo
-
                 # the exclusive check lock keeps the pack set fixed, so re-listing packs/ inside
                 # build_chunkindex_from_repo matches this verification. write_immediately persists the
                 # index and drops the corrupt fragments.
@@ -1240,7 +1238,7 @@ def recorded_ts(info):
             logger.info(f"{done} {mode} repository check, no problems found{so_far}.")
         elif not repair:
             logger.error(f"{done} {mode} repository check, errors found{so_far}.")
-        elif index_repaired and not (pack_errors or corrupt_ids):
+        elif index_repaired and not (pack_errors or corrupt_ids or missing_pack_ids):
             # the index was the only problem and it has been rebuilt from the packs.
             logger.info(f"{done} {mode} repository check, repaired{so_far}.")
         elif pack_errors or corrupt_ids:
@@ -1253,16 +1251,19 @@ def recorded_ts(info):
                 # a full check's archives phase reads archive/item metadata (and file content with
                 # --verify-data), so it repairs a corrupt pack holding such objects; warn rather than fail.
                 logger.warning(f"{done} {mode} repository check, corrupt pack(s) found{so_far}.")
-        else:
+        elif index_errors and not index_repaired:
             # the index is corrupt but was not rebuilt, e.g. the pack verification was interrupted
             # before every pack was confirmed intact; the corrupt index is left in place.
             logger.error(f"{done} {mode} repository check, index still corrupt{so_far}.")
-        # in repair mode a corrupt index left unrebuilt is a failure; a corrupt pack fails only a
-        # repository-only run, while a full check defers it to the archives phase.
+        else:
+            # index-referenced packs are missing, so their chunks are lost.
+            logger.error(f"{done} {mode} repository check, errors found{so_far}.")
+        # in repair mode a corrupt index left unrebuilt is a failure; a corrupt or missing pack fails
+        # only a repository-only run, while a full check defers it to the archives phase.
         if repair:
             if index_errors and not index_repaired:
                 return False
-            return not (repo_only and (pack_errors or corrupt_ids))
+            return not (repo_only and (pack_errors or corrupt_ids or missing_pack_ids))
         return not problems
 
     def list(self, limit=None, marker=None):

From 9bf6d5ce9acf93af5217ccf8351734d869d3b8ea Mon Sep 17 00:00:00 2001
From: Mrityunjay Raj <mr.raj.earth@gmail.com>
Date: Sat, 15 Aug 2026 18:54:18 +0530
Subject: [PATCH 137/144] check --repair: test the interrupted-rebuild guard
 and missing-pack error path

---
 src/borg/testsuite/repository_test.py | 47 +++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py
index c9020dd851..2352e5ad79 100644
--- a/src/borg/testsuite/repository_test.py
+++ b/src/borg/testsuite/repository_test.py
@@ -1132,6 +1132,53 @@ def test_check_repair_refuses_when_pack_corrupt(tmp_path):
         assert repository.check(repair=False) is False  # index was not rebuilt; still corrupt
 
 
+def test_check_repair_leaves_index_when_interrupted(tmp_path, caplog, monkeypatch):
+    # an interrupted repair (SIGINT before every pack is verified) must not rebuild the index from
+    # packs it did not confirm intact: it leaves the corrupt index in place and fails.
+    location = os.fspath(tmp_path / "repo")
+    ids = [H(x) for x in range(10)]
+    with Repository(location, exclusive=True, create=True) as repository:
+        for i, cid in enumerate(ids):
+            repository.put(cid, fchunk(bytes([i]) * 20, chunk_id=cid))
+        repository.flush()  # seal the pack(s) and let close() persist the index
+    with reopen(repository) as repository:
+        for info in repository.store_list("index"):  # rot every fragment so repair takes the rebuild path
+            name = f"index/{info.name}"
+            data = bytearray(repository.store_load(name))
+            data[0] ^= 0xFF
+            repository.store_store(name, bytes(data))
+    with reopen(repository) as repository:
+        monkeypatch.setattr("borg.repository.sig_int", True)  # simulate a SIGINT before the pack loop
+        with caplog.at_level(logging.ERROR, logger="borg.repository"):
+            assert repository.check(repair=True) is False  # interrupted: index not rebuilt, so it fails
+        assert "index still corrupt" in caplog.text
+    with reopen(repository) as repository:
+        assert repository.check(repair=False) is False  # repair left the index corrupt
+
+
+def test_check_repair_reports_missing_pack_as_error(tmp_path, caplog):
+    # a repair with an intact index but a pack the index references missing from packs/ reports the
+    # loss and fails a repository-only run; a full check defers it to the archives phase (refs #9898,
+    # #8572).
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()  # flush before close persists the index
+    with reopen(repository) as repository:
+        pack_id = repository.chunks[H(0)].pack_id
+        repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone, index entry kept
+    with reopen(repository) as repository:
+        # a repository-only repair cannot recover the lost chunks, so it fails and reports the error.
+        with caplog.at_level(logging.ERROR, logger="borg.repository"):
+            assert repository.check(repair=True, repo_only=True) is False
+        assert f"Missing pack: {bin_to_hex(pack_id)}" in caplog.text
+        assert "errors found" in caplog.text
+    with reopen(repository) as repository:
+        # a full check defers the missing pack to the archives phase, so the repository phase passes.
+        assert repository.check(repair=True, repo_only=False) is True
+
+
 def test_check_warns_on_invalid_chunk_index(tmp_path, caplog):
     # check warns about an invalid chunk index but does not fail, since the index is not part of
     # the repository's object integrity.

From 7433e1c4c4d4fd15590a3498f7b1c98224026d35 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 15 Aug 2026 12:42:36 +0200
Subject: [PATCH 138/144] create: follow symlinks given as recursion roots,
 fixes #4737

If a recursion root (a path given on the command line or via a patterns
file) is a symlink, borg now follows it and archives what it points to,
using the path as given.

Symlinks encountered while recursing are not affected, they are archived
as symlinks as before. Symlinks given via --paths-from-* are never
followed either.

Archiving the symlink target under the symlink's path also keeps the files
cache working if the symlink target changes its name (e.g. a 'current'
symlink pointing to the latest of a series of directories).

A recursion root that is a symlink with a non-existing target is skipped
with a warning (new BackupBrokenSymlinkError, rc 112).

As a followed symlink must be opened without O_NOFOLLOW, add flags_dir_follow
and flags_normal_follow and let the callers choose the flags.
---
 docs/internals/frontends.rst                  |   2 +
 docs/man/borg-create.1                        |  50 +++++++-
 docs/usage/create.rst.inc                     |  22 +++-
 src/borg/archive.py                           |  10 +-
 src/borg/archiver/create_cmd.py               | 106 ++++++++++++++---
 src/borg/helpers/__init__.py                  |   4 +-
 src/borg/helpers/errors.py                    |   8 ++
 src/borg/helpers/fs.py                        |   2 +
 .../testsuite/archiver/create_cmd_test.py     | 110 +++++++++++++++++-
 9 files changed, 288 insertions(+), 26 deletions(-)

diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst
index f3162735fc..fea84ca753 100644
--- a/docs/internals/frontends.rst
+++ b/docs/internals/frontends.rst
@@ -819,6 +819,8 @@ Warnings
         {}: {}
     BackupTimeoutError rc: 111
         {}: {}
+    BackupBrokenSymlinkError rc: 112
+        {}: {}
 
 Operations
     - cache.begin_transaction
diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1
index a5fc9ece74..7729fbbfad 100644
--- a/docs/man/borg-create.1
+++ b/docs/man/borg-create.1
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "borg-create" "1" "2026-08-13" "" "borg backup tool"
+.TH "borg-create" "1" "2026-08-15" "" "borg backup tool"
 .SH Name
 borg-create \- Creates a new archive.
 .SH SYNOPSIS
@@ -46,6 +46,23 @@ The slashdot hack in paths (recursion roots) is triggered by using \fB/./\fP:
 strip the prefix on the left side of \fB\&./\fP from the archived items (in this case,
 \fBthis/gets/archived\fP will be the path in the archived item).
 .sp
+If a recursion root (a path given on the command line or in a patterns file) is a
+symlink, borg follows it and archives what it points to \- using the path you gave.
+If \fBcurrent\fP is a symlink pointing to the directory \fB20260801\-2345\fP,
+\fBborg create ARCHIVE current\fP thus archives \fBcurrent\fP as a directory (with the
+metadata of \fB20260801\-2345\fP) and recurses into it, archiving the contained fs
+objects as \fBcurrent/...\fP\&. As the archived paths do not change when the symlink
+target changes, the files cache keeps working for such backups.
+.sp
+Note that the symlink itself is then not in the archive (and neither is its target
+path), so restoring will create a real directory (or file) where the symlink was.
+If you want the symlink archived as a symlink, do not give it as a recursion root,
+but let borg find it while recursing (symlinks found that way are never followed).
+A recursion root that is a symlink with a non\-existing target is skipped with a warning.
+.sp
+If you give both a symlink and its target as recursion roots, borg archives the fs
+objects only once, under the path given first (like for any other root given twice).
+.sp
 When specifying \(aq\-\(aq as a path, borg will read data from standard input and create a
 file named \(aqstdin\(aq in the created archive from that data. In some cases, it is more
 appropriate to use \-\-content\-from\-command. See the section \fIReading from stdin\fP
@@ -71,9 +88,9 @@ the files cache.
 This comparison can operate in different modes as given by \fB\-\-files\-cache\fP:
 .INDENT 0.0
 .IP \(bu 2
-ctime,size,inode (default)
+ctime,size,inode (default on POSIX systems)
 .IP \(bu 2
-mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4)
+mtime,size,inode (default on Windows)
 .IP \(bu 2
 ctime,size (ignore the inode number)
 .IP \(bu 2
@@ -109,6 +126,13 @@ can be arbitrarily set from userspace, e.g., to set mtime back to the same value
 it had before a content change happened. This can be used maliciously as well as
 well\-meant, but in both cases mtime\-based cache modes can be problematic.
 .UNINDENT
+.sp
+On Windows, ctime is the file \fIcreation\fP time, not the \(dqmetadata change time\(dq it is
+on POSIX systems. A ctime based mode would therefore not notice content changes of a
+file that keeps its size and inode number, so borg defaults to \fBmtime,size,inode\fP
+there. If a ctime based mode is given explicitly on Windows, borg warns and uses the
+corresponding mtime based mode instead: ctime,size,inode \-> mtime,size,inode,
+ctime,size \-> mtime,size, rechunk,ctime \-> rechunk,mtime.
 .INDENT 0.0
 .TP
 .B The \fB\-\-files\-changed\fP option controls how Borg detects if a file has changed during backup:
@@ -134,11 +158,22 @@ The \fB\-\-progress\fP option shows (from left to right) Original and (uncompres
 deduplicated size (O and U respectively), then the Number of files (N) processed so far,
 followed by the currently processed path.
 .sp
+Sizes of GB and above are shown with enough decimal places that even MB\-sized progress
+stays visible. On a terminal, this needs a width of at least 110 columns \- on narrower
+terminals, the compact format is used, so that the path stays readable. If the output
+does not go to a terminal (e.g. into a logfile), the precise format is always used.
+.sp
 When using \fB\-\-stats\fP, you will get some statistics about how much data was
 added \- the \(dqThis Archive\(dq deduplicated size there is most interesting as that is
 how much your repository will grow. Please note that the \(dqAll archives\(dq stats refer to
-the state after creation. Also, the \fB\-\-stats\fP and \fB\-\-dry\-run\fP options are mutually
-exclusive because the data is not actually compressed and deduplicated during a dry run.
+the state after creation.
+.sp
+When \fB\-\-stats\fP is used together with \fB\-\-dry\-run\fP, only the number of files and the
+original size are reported. They are computed from file system metadata, without reading
+the file contents, so a dry run stays fast. As data is not actually read, chunked, and
+deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read
+from standard input, from a command\(aqs output, or from special files (\fB\-\-read\-special\fP)
+are also unknown in a dry run and counted as zero.
 .sp
 The \fB\-\-stats\fP output also reports the store statistics (lines prefixed with
 \(dqStore\(dq), taken from the storage layer after this run. These cover the backend and
@@ -275,7 +310,7 @@ do not read and store xattrs into archive
 detect sparse holes in input (supported only by fixed chunker)
 .TP
 .BI \-\-files\-cache \ MODE
-operate files cache in MODE. default: ctime,size,inode
+operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there).
 .TP
 .BI \-\-files\-changed \ MODE
 specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).
@@ -559,6 +594,9 @@ to borg (maybe implementing your own recursion or your own rules), you can use
 .sp
 Borg supports paths with the slashdot hack to strip path prefixes here also.
 So, be careful not to unintentionally trigger that.
+.sp
+Symlinks given this way are never followed (unlike recursion roots are), they are
+archived as symlinks.
 .SH SEE ALSO
 .sp
 \fIborg\-common(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-check(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP, \fIborg\-compression(1)\fP, \fIborg\-repo\-create(1)\fP
diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc
index 60a10c10c1..268b6387f1 100644
--- a/docs/usage/create.rst.inc
+++ b/docs/usage/create.rst.inc
@@ -196,6 +196,23 @@ The slashdot hack in paths (recursion roots) is triggered by using ``/./``:
 strip the prefix on the left side of ``./`` from the archived items (in this case,
 ``this/gets/archived`` will be the path in the archived item).
 
+If a recursion root (a path given on the command line or in a patterns file) is a
+symlink, borg follows it and archives what it points to - using the path you gave.
+If ``current`` is a symlink pointing to the directory ``20260801-2345``,
+``borg create ARCHIVE current`` thus archives ``current`` as a directory (with the
+metadata of ``20260801-2345``) and recurses into it, archiving the contained fs
+objects as ``current/...``. As the archived paths do not change when the symlink
+target changes, the files cache keeps working for such backups.
+
+Note that the symlink itself is then not in the archive (and neither is its target
+path), so restoring will create a real directory (or file) where the symlink was.
+If you want the symlink archived as a symlink, do not give it as a recursion root,
+but let borg find it while recursing (symlinks found that way are never followed).
+A recursion root that is a symlink with a non-existing target is skipped with a warning.
+
+If you give both a symlink and its target as recursion roots, borg archives the fs
+objects only once, under the path given first (like for any other root given twice).
+
 When specifying '-' as a path, borg will read data from standard input and create a
 file named 'stdin' in the created archive from that data. In some cases, it is more
 appropriate to use --content-from-command. See the section *Reading from stdin*
@@ -441,4 +458,7 @@ to borg (maybe implementing your own recursion or your own rules), you can use
 (with the latter two, borg will fail to create an archive should the command fail).
 
 Borg supports paths with the slashdot hack to strip path prefixes here also.
-So, be careful not to unintentionally trigger that.
\ No newline at end of file
+So, be careful not to unintentionally trigger that.
+
+Symlinks given this way are never followed (unlike recursion roots are), they are
+archived as symlinks.
\ No newline at end of file
diff --git a/src/borg/archive.py b/src/borg/archive.py
index db7de5cd03..cd9c65f5cc 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -1454,7 +1454,7 @@ def process_dir(self, *, path, parent_fd, name, st, strip_prefix):
                 item.update(self.metadata_collector.stat_attrs(st, path, fd=fd))
                 return status
 
-    def process_fifo(self, *, path, parent_fd, name, st, strip_prefix):
+    def process_fifo(self, *, path, parent_fd, name, st, strip_prefix, flags=flags_normal):
         with self.create_helper(path, st, "f", strip_prefix=strip_prefix) as (
             item,
             status,
@@ -1463,13 +1463,13 @@ def process_fifo(self, *, path, parent_fd, name, st, strip_prefix):
         ):  # fifo
             if item is None:
                 return status
-            with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_normal, noatime=True) as fd:
+            with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags, noatime=True) as fd:
                 with backup_io("fstat"):
                     st = stat_update_check(st, os.fstat(fd))
                 item.update(self.metadata_collector.stat_attrs(st, path, fd=fd))
                 return status
 
-    def process_dev(self, *, path, parent_fd, name, st, dev_type, strip_prefix):
+    def process_dev(self, *, path, parent_fd, name, st, dev_type, strip_prefix, follow_symlinks=False):
         with self.create_helper(path, st, dev_type, strip_prefix=strip_prefix) as (
             item,
             status,
@@ -1480,7 +1480,9 @@ def process_dev(self, *, path, parent_fd, name, st, dev_type, strip_prefix):
             if item is None:
                 return status
             with backup_io("stat"):
-                st = stat_update_check(st, os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False))
+                st = stat_update_check(
+                    st, os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=follow_symlinks)
+                )
             item.rdev = st.st_rdev
             item.update(self.metadata_collector.stat_attrs(st, path))
             return status
diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py
index 93d316063a..5dbc2ec455 100644
--- a/src/borg/archiver/create_cmd.py
+++ b/src/borg/archiver/create_cmd.py
@@ -21,10 +21,12 @@
 from ..helpers import eval_escapes
 from ..helpers import timestamp, archive_ts_now
 from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify
+from ..helpers import BackupBrokenSymlinkError
 from ..helpers import dir_is_tagged
 from ..helpers import log_multi
 from ..helpers import basic_json_data, json_print, FileSize
-from ..helpers import flags_dir, flags_special_follow, flags_special
+from ..helpers import flags_dir, flags_dir_follow, flags_special_follow, flags_special
+from ..helpers import flags_normal, flags_normal_follow
 from ..helpers import prepare_subprocess_env
 from ..helpers import sig_int, ignore_sigint
 from ..helpers import iter_separated
@@ -40,6 +42,24 @@
 logger = create_logger()
 
 
+def stat_root(path):
+    """
+    stat a recursion root, following it if it is a symlink, see #4737.
+
+    Returns (st, followed): for a symlink, st is the stat of what it points to and followed
+    is True, so the caller knows that it must not use O_NOFOLLOW when opening path.
+
+    Raises BackupBrokenSymlinkError if path is a symlink with a non-existing target.
+    """
+    st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
+    if not stat.S_ISLNK(st.st_mode):
+        return st, False
+    try:
+        return os_stat(path=path, parent_fd=None, name=None, follow_symlinks=True), True
+    except FileNotFoundError:
+        raise BackupBrokenSymlinkError("stat", "broken symlink, skipping it") from None
+
+
 class CreateMixIn:
     @with_repository(compatibility=(Manifest.Operation.WRITE,))
     def do_create(self, args, repository, manifest):
@@ -144,6 +164,7 @@ def create_inner(archive, cache, fso):
                     path = posixpath.normpath(path)
                     try:
                         with backup_io("stat"):
+                            # symlinks given this way are never followed, see #4737.
                             st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
                         status = self._process_any(
                             path=path,
@@ -199,7 +220,7 @@ def create_inner(archive, cache, fso):
                     path = posixpath.normpath(path)
                     try:
                         with backup_io("stat"):
-                            st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
+                            st, followed = stat_root(path)
                         restrict_dev = st.st_dev if args.one_file_system else None
                         self._rec_walk(
                             path=path,
@@ -216,6 +237,7 @@ def create_inner(archive, cache, fso):
                             read_special=args.read_special,
                             dry_run=dry_run,
                             strip_prefix=strip_prefix,
+                            follow_symlink=followed,
                         )
                         # if we get back here, we've finished recursing into <path>,
                         # we do not ever want to get back in there (even if path is given twice as recursion root)
@@ -324,9 +346,14 @@ def create_inner(archive, cache, fso):
                         logger=logging.getLogger("borg.output.stats"),
                     )
 
-    def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, dry_run, strip_prefix):
+    def _process_any(
+        self, *, path, parent_fd, name, st, fso, cache, read_special, dry_run, strip_prefix, followed_symlink=False
+    ):
         """
         Call the right method on the given FilesystemObjectProcessor.
+
+        If followed_symlink is True, *path* is a symlink we followed (recursion root, see #4737)
+        and *st* is the stat of its target, so we must not use O_NOFOLLOW when opening it.
         """
 
         if dry_run:
@@ -348,6 +375,9 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                     stats.nfiles += 1  # size unknown without reading the special file
             return "+"  # included
         MAX_RETRIES = 10  # count includes the initial try (initial try == "retry 0")
+        # if we followed a symlink, we must not refuse to open its target via the symlink:
+        flags_file = flags_normal_follow if followed_symlink else flags_normal
+        flags_specialfile = flags_special_follow if followed_symlink else flags_special
         for retry in range(MAX_RETRIES):
             last_try = retry == MAX_RETRIES - 1
             try:
@@ -358,10 +388,12 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                         name=name,
                         st=st,
                         cache=cache,
+                        flags=flags_file,
                         last_try=last_try,
                         strip_prefix=strip_prefix,
                     )
                 elif stat.S_ISDIR(st.st_mode):
+                    # note: a followed symlink to a directory does not get here, _rec_walk deals with it.
                     return fso.process_dir(path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix)
                 elif stat.S_ISLNK(st.st_mode):
                     if not read_special:
@@ -393,7 +425,12 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                 elif stat.S_ISFIFO(st.st_mode):
                     if not read_special:
                         return fso.process_fifo(
-                            path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
+                            path=path,
+                            parent_fd=parent_fd,
+                            name=name,
+                            st=st,
+                            strip_prefix=strip_prefix,
+                            flags=flags_file,
                         )
                     else:
                         return fso.process_file(
@@ -402,14 +439,20 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                             name=name,
                             st=st,
                             cache=cache,
-                            flags=flags_special,
+                            flags=flags_specialfile,
                             last_try=last_try,
                             strip_prefix=strip_prefix,
                         )
                 elif stat.S_ISCHR(st.st_mode):
                     if not read_special:
                         return fso.process_dev(
-                            path=path, parent_fd=parent_fd, name=name, st=st, dev_type="c", strip_prefix=strip_prefix
+                            path=path,
+                            parent_fd=parent_fd,
+                            name=name,
+                            st=st,
+                            dev_type="c",
+                            strip_prefix=strip_prefix,
+                            follow_symlinks=followed_symlink,
                         )
                     else:
                         return fso.process_file(
@@ -418,14 +461,20 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                             name=name,
                             st=st,
                             cache=cache,
-                            flags=flags_special,
+                            flags=flags_specialfile,
                             last_try=last_try,
                             strip_prefix=strip_prefix,
                         )
                 elif stat.S_ISBLK(st.st_mode):
                     if not read_special:
                         return fso.process_dev(
-                            path=path, parent_fd=parent_fd, name=name, st=st, dev_type="b", strip_prefix=strip_prefix
+                            path=path,
+                            parent_fd=parent_fd,
+                            name=name,
+                            st=st,
+                            dev_type="b",
+                            strip_prefix=strip_prefix,
+                            follow_symlinks=followed_symlink,
                         )
                     else:
                         return fso.process_file(
@@ -434,7 +483,7 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                             name=name,
                             st=st,
                             cache=cache,
-                            flags=flags_special,
+                            flags=flags_specialfile,
                             last_try=last_try,
                             strip_prefix=strip_prefix,
                         )
@@ -469,7 +518,7 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
                 # mode right (which could have changed due to a race condition and is important for
                 # dispatching) and also to get current inode number of that file.
                 with backup_io("stat"):
-                    st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
+                    st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=followed_symlink)
 
     def _rec_walk(
         self,
@@ -488,10 +537,15 @@ def _rec_walk(
         read_special,
         dry_run,
         strip_prefix,
+        follow_symlink=False,
     ):
         """
         Process *path* (or, preferably, parent_fd/name) recursively according to the various parameters.
 
+        follow_symlink is only given for a recursion root that is a symlink, see #4737: we then
+        archive what the symlink points to (using the symlink's path). Symlinks encountered while
+        recursing are never followed, so this is never passed on to the recursive calls.
+
         This should only raise on critical errors. Per-item errors must be handled within this method.
         """
         if sig_int and sig_int.action_done():
@@ -503,7 +557,7 @@ def _rec_walk(
             recurse_excluded_dir = False
             if matcher.match(path):
                 with backup_io("stat"):
-                    st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
+                    st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=follow_symlink)
             else:
                 self.print_file_status("-", path)  # excluded
                 # get out here as quickly as possible:
@@ -514,7 +568,7 @@ def _rec_walk(
                     return
                 recurse_excluded_dir = True
                 with backup_io("stat"):
-                    st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
+                    st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=follow_symlink)
                 if not stat.S_ISDIR(st.st_mode):
                     return
 
@@ -547,10 +601,16 @@ def _rec_walk(
                     read_special=read_special,
                     dry_run=dry_run,
                     strip_prefix=strip_prefix,
+                    followed_symlink=follow_symlink,
                 )
             else:
                 with OsOpen(
-                    path=path, parent_fd=parent_fd, name=name, flags=flags_dir, noatime=True, op="dir_open"
+                    path=path,
+                    parent_fd=parent_fd,
+                    name=name,
+                    flags=flags_dir_follow if follow_symlink else flags_dir,
+                    noatime=True,
+                    op="dir_open",
                 ) as child_fd:
                     # child_fd is None for directories on windows, in that case a race condition check is not possible.
                     if child_fd is not None:
@@ -646,6 +706,23 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         strip the prefix on the left side of ``./`` from the archived items (in this case,
         ``this/gets/archived`` will be the path in the archived item).
 
+        If a recursion root (a path given on the command line or in a patterns file) is a
+        symlink, borg follows it and archives what it points to - using the path you gave.
+        If ``current`` is a symlink pointing to the directory ``20260801-2345``,
+        ``borg create ARCHIVE current`` thus archives ``current`` as a directory (with the
+        metadata of ``20260801-2345``) and recurses into it, archiving the contained fs
+        objects as ``current/...``. As the archived paths do not change when the symlink
+        target changes, the files cache keeps working for such backups.
+
+        Note that the symlink itself is then not in the archive (and neither is its target
+        path), so restoring will create a real directory (or file) where the symlink was.
+        If you want the symlink archived as a symlink, do not give it as a recursion root,
+        but let borg find it while recursing (symlinks found that way are never followed).
+        A recursion root that is a symlink with a non-existing target is skipped with a warning.
+
+        If you give both a symlink and its target as recursion roots, borg archives the fs
+        objects only once, under the path given first (like for any other root given twice).
+
         When specifying '-' as a path, borg will read data from standard input and create a
         file named 'stdin' in the created archive from that data. In some cases, it is more
         appropriate to use --content-from-command. See the section *Reading from stdin*
@@ -892,6 +969,9 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
 
         Borg supports paths with the slashdot hack to strip path prefixes here also.
         So, be careful not to unintentionally trigger that.
+
+        Symlinks given this way are never followed (unlike recursion roots are), they are
+        archived as symlinks.
         """
         )
 
diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py
index 898e92a59a..eb2a976098 100644
--- a/src/borg/helpers/__init__.py
+++ b/src/borg/helpers/__init__.py
@@ -18,12 +18,14 @@
 from .errors import BackupError, BackupOSError, BackupRaceConditionError, BackupItemExcluded
 from .errors import BackupPermissionError, BackupIOError, BackupFileNotFoundError, BackupTimeoutError
 from .errors import BackupSymlinkParentError, BackupPathTraversalError, BackupHardlinkSourceError
+from .errors import BackupBrokenSymlinkError
 from .fs import ensure_dir, join_base_dir
 from .fs import get_security_dir, get_keys_dir, get_base_dir, get_cache_dir, get_config_dir, get_runtime_dir
 from .fs import dir_is_tagged, dir_is_cachedir, remove_dotdot_prefixes, make_path_safe, scandir_inorder
 from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount, slashify
 from .fs import SpecialFileReader
-from .fs import O_, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime
+from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special
+from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime
 from .fs import HardLinkManager
 from .misc import sysinfo, log_multi, consume
 from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
diff --git a/src/borg/helpers/errors.py b/src/borg/helpers/errors.py
index 72993efb7b..b83fc43aa2 100644
--- a/src/borg/helpers/errors.py
+++ b/src/borg/helpers/errors.py
@@ -225,5 +225,13 @@ class BackupTimeoutError(BackupOSError):
     exit_mcode = 111
 
 
+class BackupBrokenSymlinkError(BackupError):
+    """{}: {}"""
+
+    # A recursion root that is a symlink pointing to a non-existing target: as we follow
+    # symlinked recursion roots, there is nothing to archive there, see #4737.
+    exit_mcode = 112
+
+
 class BackupItemExcluded(Exception):
     """Used internally to skip an item from processing when it is excluded."""
diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py
index 1412416450..653371a1b5 100644
--- a/src/borg/helpers/fs.py
+++ b/src/borg/helpers/fs.py
@@ -505,8 +505,10 @@ def O_(*flags):
 flags_special = flags_base | O_("NOFOLLOW")  # BLOCK == wait when reading devices or fifos
 flags_special_follow = flags_base  # BLOCK == wait when reading symlinked devices or fifos
 flags_normal = flags_base | O_("NONBLOCK", "NOFOLLOW")
+flags_normal_follow = flags_base | O_("NONBLOCK")  # follow a symlinked recursion root, see #4737
 flags_noatime = flags_normal | O_("NOATIME")
 flags_dir = O_("DIRECTORY", "RDONLY", "NOFOLLOW")
+flags_dir_follow = O_("DIRECTORY", "RDONLY")  # follow a symlinked recursion root, see #4737
 
 
 def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py
index c53c70a4e1..9a34c6e96d 100644
--- a/src/borg/testsuite/archiver/create_cmd_test.py
+++ b/src/borg/testsuite/archiver/create_cmd_test.py
@@ -16,7 +16,7 @@
 from ...platform import is_win32, is_cygwin
 from ...platformflags import is_msystem
 from ...repository import Repository
-from ...helpers import CommandError, BackupPermissionError, BackupTimeoutError
+from ...helpers import CommandError, BackupPermissionError, BackupTimeoutError, BackupBrokenSymlinkError
 from .. import has_lchflags, has_mknod
 from .. import changedir
 from .. import (
@@ -1205,6 +1205,114 @@ def test_create_read_special_broken_symlink(archivers, request):
     assert "input/link -> somewhere does not exist" in output
 
 
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_root_dir(archivers, request):
+    # a recursion root that is a symlink to a directory is followed, see #4737
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "target/file", contents=b"content")
+    os.symlink("target", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/link")
+    output = cmd(archiver, "list", "test")
+    assert "input/link -> target" not in output  # not archived as a symlink, but as a directory
+    assert "input/link/file" in output  # we recursed into the symlink target
+    assert "input/target" not in output  # the target path is not in the archive
+    with changedir("output"):
+        cmd(archiver, "extract", "test")
+        assert not os.path.islink("input/link")
+        assert os.path.isdir("input/link")
+        with open("input/link/file", "rb") as f:
+            assert f.read() == b"content"
+
+
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_root_file(archivers, request):
+    # a recursion root that is a symlink to a regular file is followed, see #4737
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "target", contents=b"content")
+    os.symlink("target", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/link")
+    output = cmd(archiver, "list", "test")
+    assert "input/link -> target" not in output  # not archived as a symlink, but as a regular file
+    assert "input/link" in output
+    with changedir("output"):
+        cmd(archiver, "extract", "test")
+        assert not os.path.islink("input/link")
+        with open("input/link", "rb") as f:
+            assert f.read() == b"content"
+
+
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_below_root_not_followed(archivers, request):
+    # only recursion roots are followed, symlinks found while recursing are not, see #4737
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "target/file", contents=b"content")
+    os.symlink("target", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input")
+    output = cmd(archiver, "list", "test")
+    assert "input/link -> target" in output
+    assert "input/link/file" not in output
+    assert "input/target/file" in output
+
+
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_root_and_target(archivers, request):
+    # a followed symlink root and its target are the same fs objects, so they are archived
+    # only once, under the path given first (like any other recursion root given twice).
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "target/file", contents=b"content")
+    os.symlink("target", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/link", "input/target")
+    output = cmd(archiver, "list", "test")
+    assert "input/link/file" in output
+    assert "input/target/file" not in output
+
+
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_root_broken(archivers, request):
+    # a recursion root that is a symlink with a non-existing target is skipped with a warning
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "file", contents=b"content")
+    os.symlink("somewhere does not exist", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    expected_ec = BackupBrokenSymlinkError("stat", "broken symlink, skipping it").exit_code
+    if expected_ec == EXIT_ERROR:  # workaround, TODO: fix it
+        expected_ec = EXIT_WARNING
+    out = cmd(archiver, "create", "test", "input/link", "input/file", exit_code=expected_ec)
+    assert "input/link: stat: broken symlink, skipping it" in out
+    output = cmd(archiver, "list", "test")
+    assert "input/link" not in output
+    assert "input/file" in output  # the other recursion root was archived
+
+
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_paths_from_stdin_not_followed(archivers, request):
+    # only recursion roots are followed, paths fed in via --paths-from-* are not, see #4737
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "target", contents=b"content")
+    os.symlink("target", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "--paths-from-stdin", "test", input=b"input/link")
+    output = cmd(archiver, "list", "test")
+    assert "input/link -> target" in output
+
+
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_root_dotslash_hack(archivers, request):
+    # the slashdot hack also works for a recursion root that is a symlink, see #4737
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "target/file", contents=b"content")
+    os.symlink("target", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/link/./")  # hack!
+    output = cmd(archiver, "list", "test")
+    assert "input" not in output  # the prefix left of the slashdot was stripped
+    assert "file" in output
+
+
 def test_create_dotslash_hack(archivers, request):
     archiver = request.getfixturevalue(archivers)
     os.makedirs(os.path.join(archiver.input_path, "first", "secondA", "thirdA"))

From d7855ade11f8cb4484ee2cda8d0ba8616ed671f4 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sat, 15 Aug 2026 16:46:45 +0200
Subject: [PATCH 139/144] tests: expect the warning's exit code, not the
 error's

The BackupErrors raised while backing up a fs object do not reach the
top level, they are wrapped into a BackupWarning and only reported as a
warning. Thus, the expected exit code must come from the BackupWarning
(giving EXIT_WARNING for BORG_EXIT_CODES=classic and the error's specific
warning rc for modern), not from the wrapped error (which would give
EXIT_ERROR for classic).

Removes the "workaround, TODO: fix it" that patched up EXIT_ERROR after
the fact.
---
 src/borg/testsuite/archiver/create_cmd_test.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py
index 9a34c6e96d..c8d8077f22 100644
--- a/src/borg/testsuite/archiver/create_cmd_test.py
+++ b/src/borg/testsuite/archiver/create_cmd_test.py
@@ -17,6 +17,7 @@
 from ...platformflags import is_msystem
 from ...repository import Repository
 from ...helpers import CommandError, BackupPermissionError, BackupTimeoutError, BackupBrokenSymlinkError
+from ...helpers import BackupWarning
 from .. import has_lchflags, has_mknod
 from .. import changedir
 from .. import (
@@ -322,9 +323,8 @@ def test_create_no_permission_file(archivers, request):
         os.chmod(file_path + "2", 0o000)
     cmd(archiver, "repo-create", RK_ENCRYPTION)
     flist = "".join(f"input/file{n}\n" for n in range(1, 4))
-    expected_ec = BackupPermissionError("open", OSError(13, "permission denied")).exit_code
-    if expected_ec == EXIT_ERROR:  # workaround, TODO: fix it
-        expected_ec = EXIT_WARNING
+    exc = BackupPermissionError("open", OSError(13, "permission denied"))
+    expected_ec = BackupWarning("input/file2", exc).exit_code
     out = cmd(
         archiver,
         "create",
@@ -1076,9 +1076,8 @@ def test_create_read_special_timeout_expired(archivers, request):
     cmd(archiver, "repo-create", RK_ENCRYPTION)
     create_regular_file(archiver.input_path, "file1", size=1024)
     os.mkfifo(os.path.join(archiver.input_path, "fifo"))
-    expected_ec = BackupTimeoutError("read", OSError(errno.ETIMEDOUT, "timeout")).exit_code
-    if expected_ec == EXIT_ERROR:  # workaround, TODO: fix it
-        expected_ec = EXIT_WARNING
+    exc = BackupTimeoutError("read", OSError(errno.ETIMEDOUT, "timeout"))
+    expected_ec = BackupWarning("input/fifo", exc).exit_code
     started = time.monotonic()
     out = cmd(
         archiver,
@@ -1278,9 +1277,8 @@ def test_create_symlink_root_broken(archivers, request):
     create_regular_file(archiver.input_path, "file", contents=b"content")
     os.symlink("somewhere does not exist", os.path.join(archiver.input_path, "link"))
     cmd(archiver, "repo-create", RK_ENCRYPTION)
-    expected_ec = BackupBrokenSymlinkError("stat", "broken symlink, skipping it").exit_code
-    if expected_ec == EXIT_ERROR:  # workaround, TODO: fix it
-        expected_ec = EXIT_WARNING
+    exc = BackupBrokenSymlinkError("stat", "broken symlink, skipping it")
+    expected_ec = BackupWarning("input/link", exc).exit_code
     out = cmd(archiver, "create", "test", "input/link", "input/file", exit_code=expected_ec)
     assert "input/link: stat: broken symlink, skipping it" in out
     output = cmd(archiver, "list", "test")

From 91b030e4862a10f37f52692c063b8bcea7e1dd1f Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 16 Aug 2026 11:28:19 +0200
Subject: [PATCH 140/144] tests: do not pin the zsh --sort-by label, shtab
 1.11.0 changed it

shtab 1.11.0 (released 2026-08-15) labels a zsh option argument with the
argument's metavar instead of its dest, so the generated spec for
"borg list --sort-by" (metavar KEYS, dest sort_by) changed from

  "--sort-by[...]:sort_by:_borg_complete_item_sortby"

to

  "--sort-by[...]:KEYS:_borg_complete_item_sortby"

test_zsh_sortby_wiring matched the ":sort_by:" form literally and started
failing for list and repo-list (borg diff --sort-by has no metavar, so
shtab still falls back to the dest there and that entry kept passing).

The completions are fine: the label is only what zsh displays while
completing, and --sort-by is still wired to our helper function. Match the
helper and leave the label open, so either shtab behaviour passes.

Note that shtab is not pinned in requirements.d/, it comes in via the
shtab>=1.9.3 dependency in pyproject.toml, so CI picks up new shtab
releases as they appear.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/borg/testsuite/archiver/completion_cmd_test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/borg/testsuite/archiver/completion_cmd_test.py b/src/borg/testsuite/archiver/completion_cmd_test.py
index 1ef5add254..e45f40001f 100644
--- a/src/borg/testsuite/archiver/completion_cmd_test.py
+++ b/src/borg/testsuite/archiver/completion_cmd_test.py
@@ -279,7 +279,10 @@ def test_zsh_sortby_wiring(archivers, request, command, fn):
     lines = completion_lines(archivers, request, "zsh")
     block = lines[lines.index(f"_shtab_borg_{command.replace('-', '_')}_options=(") :]
     block = block[: block.index(")")]
-    assert any(f':sort_by:{fn}"' in line for line in block)
+    # The ":<label>:<helper>" label is only what zsh displays while completing: shtab used the
+    # argument's dest for it and uses its metavar since 1.11.0. Do not pin the label down, just
+    # require that --sort-by is wired to our helper.
+    assert any(re.search(rf'^\s*"--sort-by\[.*\]:[^:]+:{fn}"', line) for line in block)
     assert f"{fn}() {{" in lines
 
 

From db2ff309dab842cb33fe14da1bee73226a615470 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 14 Jun 2026 14:50:52 +0200
Subject: [PATCH 141/144] fix daemonizing: don't lose an early notify signal
 from the background process

The foreground process installs SIGTERM/SIGHUP/SIGINT handlers (via
archiver.run) before it reaches the point where it actually waits for the
background (grandchild) process to notify it (via os.kill). If the
background process started up and signalled fast enough, the signal was
delivered to the foreground while it was still between os.fork() and the
waiting code, so the globally installed handler raised at an unexpected,
uncaught place. The signal then escaped daemonizing(), bubbled up through
repository teardown (NotLocked) and made "borg mount" exit with rc 74.

This was observed flaky in CI with coverage's sys.monitoring backend on
Python 3.14 (its first-branch lazy source parse widens the window) and the
pyfuse3 backend (faster grandchild startup).

Fix the race by blocking the notify signals before the fork in
_daemonize() and waiting for them atomically in the foreground. An early
signal then stays pending and is reliably picked up by the wait. The
background process restores the original signal mask so it keeps normal
signal handling.

Use signal.sigwait() plus a SIGALRM timer (signal.setitimer) for the
wait, rather than signal.sigtimedwait(): the latter does not exist on
macOS, where it would raise AttributeError in the foreground and let it
die before the background migrated the lock (breaking test_migrate_lock_alive).

signal.SIGALRM in turn does not exist on Windows, where referencing it at
module import time would raise AttributeError and break the import chain
(and the Windows PyInstaller build), so guard it with hasattr, matching the
defensive getattr pattern already used for the notify signals. Daemonizing
is not supported on Windows anyway (no os.fork), so the empty SIGALRM list
has no functional effect there.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/borg/helpers/process.py | 98 ++++++++++++++++++++++---------------
 1 file changed, 59 insertions(+), 39 deletions(-)

diff --git a/src/borg/helpers/process.py b/src/borg/helpers/process.py
index b19aabcf18..a66e7afbf1 100644
--- a/src/borg/helpers/process.py
+++ b/src/borg/helpers/process.py
@@ -18,13 +18,32 @@
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_SIGNAL_BASE, Error
 
 
+# Signals the background (grandchild) process uses to notify the foreground process that
+# it has started up (or failed to start up), see daemonizing(). These are blocked across
+# the fork in _daemonize() so that an early notification can not be delivered to the
+# foreground before it is ready to wait for it - otherwise the globally installed signal
+# handlers (see archiver.run) would raise at an unexpected, uncaught place.
+DAEMONIZE_NOTIFY_SIGNALS = [
+    sig for sig in (getattr(signal, name, None) for name in ("SIGTERM", "SIGHUP", "SIGINT")) if sig is not None
+]
+# The foreground process waits for the notify signals with signal.sigwait() (which, unlike
+# signal.sigtimedwait(), is also available on macOS). To still honor the timeout, we arm a
+# SIGALRM timer and wait for it as well, so it must be blocked and waited for, too.
+# SIGALRM is not available on Windows (where daemonizing is not supported anyway).
+DAEMONIZE_WAIT_SIGNALS = DAEMONIZE_NOTIFY_SIGNALS + ([signal.SIGALRM] if hasattr(signal, "SIGALRM") else [])
+
+
 @contextlib.contextmanager
 def _daemonize():
     from ..platform import get_process_id
 
     old_id = get_process_id()
+    # Block the notify (and timeout) signals before forking, see DAEMONIZE_WAIT_SIGNALS.
+    old_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, DAEMONIZE_WAIT_SIGNALS)
     pid = os.fork()
     if pid:
+        # The foreground process keeps the notify signals blocked - daemonizing() waits
+        # for them explicitly via signal.sigwait(), so it can not miss an early one.
         exit_code = EXIT_SUCCESS
         try:
             yield old_id, None
@@ -33,6 +52,8 @@ def _daemonize():
         finally:
             logger.debug("Daemonizing: Foreground process (%s, %s, %s) is now dying." % old_id)
             os._exit(exit_code)
+    # Background process: restore the original signal mask, we want normal handling here.
+    signal.pthread_sigmask(signal.SIG_SETMASK, old_sigmask)
     os.setsid()
     pid = os.fork()
     if pid:
@@ -81,47 +102,46 @@ def daemonizing(*, timeout=5, show_rc=False):
             # The original / parent process, waiting for a signal to die.
             logger.debug("Daemonizing: Foreground process (%s, %s, %s) is waiting for background process..." % old_id)
             exit_code = EXIT_SUCCESS
-            # Indeed, SIGHUP and SIGTERM handlers should have been set on archiver.run(). Just in case...
-            with (
-                signal_handler("SIGINT", raising_signal_handler(KeyboardInterrupt)),
-                signal_handler("SIGHUP", raising_signal_handler(SigHup)),
-                signal_handler("SIGTERM", raising_signal_handler(SigTerm)),
-            ):
+            # The notify signals are blocked since before the fork in _daemonize(), so we can
+            # wait for them atomically here and will not miss an early signal from the background
+            # process (which could otherwise be delivered before we are ready to wait for it and
+            # then escape uncaught). See DAEMONIZE_WAIT_SIGNALS.
+            sighup = getattr(signal, "SIGHUP", None)
+            if timeout > 0:
+                # Arm a timer so we don't wait forever if the background never signals us.
+                # SIGALRM is blocked, too, so it does not run a handler but ends the sigwait().
+                signal.setitimer(signal.ITIMER_REAL, timeout)
                 try:
-                    if timeout > 0:
-                        time.sleep(timeout)
-                except SigTerm:
-                    # Normal termination; expected from grandchild, see 'os.kill()' below
-                    pass
-                except SigHup:
-                    # Background wants to indicate a problem; see 'os.kill()' below,
-                    # log message will come from grandchild.
-                    exit_code = EXIT_WARNING
-                except KeyboardInterrupt:
-                    # Manual termination.
-                    logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id)
-                    exit_code = EXIT_SIGNAL_BASE + 2
-                except BaseException as e:
-                    # Just in case...
-                    logger.warning(
-                        "Daemonizing: Foreground process received an exception while waiting:\n"
-                        + "".join(traceback.format_exception(e.__class__, e, e.__traceback__))
-                    )
-                    exit_code = EXIT_WARNING
-                else:
-                    logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?")
-                    exit_code = EXIT_WARNING
+                    sig = signal.sigwait(DAEMONIZE_WAIT_SIGNALS)
                 finally:
-                    # Before terminating the foreground process, honor --show-rc by logging the rc here as well.
-                    # This is mostly a consistency fix and not very useful considering that the main action
-                    # happens in the daemon process.
-                    if show_rc:
-                        from ..helpers import do_show_rc
-
-                        do_show_rc(exit_code)
-                    # Don't call with-body, but die immediately!
-                    # return would be sufficient, but we want to pass the exit code.
-                    raise _ExitCodeException(exit_code)
+                    signal.setitimer(signal.ITIMER_REAL, 0)  # disarm
+            else:
+                sig = signal.SIGALRM  # no waiting: behave as if the timeout had elapsed
+            if sig == signal.SIGALRM:
+                # Timeout: the background process did not signal us in time.
+                logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?")
+                exit_code = EXIT_WARNING
+            elif sig == signal.SIGTERM:
+                # Normal termination; expected from grandchild, see 'os.kill()' below.
+                pass
+            elif sighup is not None and sig == sighup:
+                # Background wants to indicate a problem; see 'os.kill()' below,
+                # log message will come from grandchild.
+                exit_code = EXIT_WARNING
+            elif sig == signal.SIGINT:
+                # Manual termination.
+                logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id)
+                exit_code = EXIT_SIGNAL_BASE + 2
+            # Before terminating the foreground process, honor --show-rc by logging the rc here as well.
+            # This is mostly a consistency fix and not very useful considering that the main action
+            # happens in the daemon process.
+            if show_rc:
+                from ..helpers import do_show_rc
+
+                do_show_rc(exit_code)
+            # Don't call with-body, but die immediately!
+            # return would be sufficient, but we want to pass the exit code.
+            raise _ExitCodeException(exit_code)
 
         # The background / grandchild process.
         sig_to_foreground = signal.SIGTERM

From 54b1b3d17a74746b868e5f54eee6c08d895674f9 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 14 Jun 2026 18:10:41 +0200
Subject: [PATCH 142/144] conftest: fix rmtree cleanup crash on Linux
 (os.lchflags does not exist)

The archiver fixture's rmtree onerror handler called os.lchflags(path, 0)
when has_lchflags was True. But has_lchflags is also True on Linux (flags
are cleared via ioctl there), where os.lchflags does not exist, raising an
uncaught AttributeError and turning teardown into an ERROR. Use borg's
cross-platform platform.set_flags instead.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/borg/conftest.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/borg/conftest.py b/src/borg/conftest.py
index c76cfcf626..981d288049 100644
--- a/src/borg/conftest.py
+++ b/src/borg/conftest.py
@@ -15,6 +15,7 @@
 setup_logging()
 
 from borg.archiver import Archiver  # noqa: E402
+from borg.platform import set_flags  # noqa: E402
 from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3, has_mfusepy  # noqa: E402
 from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported  # noqa: E402
 from borg.testsuite.archiver import BORG_EXES
@@ -196,8 +197,10 @@ def archiver(tmp_path, set_env_variables):
     def maybe_clear_flags_and_retry(func, path, _exc_info):
         if has_lchflags:
             # Clear any BSD flags (e.g. UF_APPEND) that may have prevented removal, then retry once.
+            # Note: use borg's cross-platform set_flags, not os.lchflags - the latter does not exist
+            # on Linux even though has_lchflags is True there (Linux clears flags via ioctl).
             try:
-                os.lchflags(path, 0)
+                set_flags(path, 0)
                 func(path)
             except OSError:
                 pass

From def938377785a20f72b2f5ceed9952c94accbe7d Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 16 Aug 2026 12:27:59 +0200
Subject: [PATCH 143/144] get rid of master/slave terminology for hard links,
 #5248

borg2 does not have the asymmetric hard link model any more (all members of a
hard link group have their own chunks list, they are identified by hlid), so
the remaining master/slave wording only described borg1 archives.

Rename the borg1 compat predicates by what they actually test:

- borg1_hardlink_master -> borg1_hardlink_with_content
- borg1_hardlink_slave -> borg1_hardlink_reference

The distinction is not a hierarchy, but whether an item carries the chunks
list or only a .source back reference to the item that does. Since #7175,
borg1 set the hardlink_master key for all hardlinkable items, so that key
does not tell them apart - only the absence of .source does. Added a test
pinning that.

The 'hardlink_master' item key itself is borg1 on-disk data and has to keep
its name, it is now commented as such.

Also fix two places that were not only using the old wording, but were also
stale: borg2 hard links do have their own chunks list and thus their real
size, they are not size 0.

While at it, fix the Item.get_size stub, it still had the long gone
hardlink_masters and compressed parameters.
---
 docs/usage/list.rst.inc                       |  2 +-
 src/borg/archiver/list_cmd.py                 |  2 +-
 src/borg/helpers/parseformat.py               |  3 +-
 src/borg/item.pyi                             | 10 +----
 src/borg/item.pyx                             |  2 +-
 src/borg/legacy/helpers.py                    | 13 ++++++-
 src/borg/legacy/upgrade.py                    |  6 +--
 .../testsuite/archiver/extract_cmd_test.py    |  2 +-
 .../testsuite/archiver/transfer_cmd_test.py   |  8 ++--
 src/borg/testsuite/legacy_helpers_test.py     | 39 +++++++++++--------
 src/borg/testsuite/legacy_upgrade_test.py     | 16 ++++----
 11 files changed, 57 insertions(+), 46 deletions(-)

diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc
index bf682d5848..39f6d2b021 100644
--- a/docs/usage/list.rst.inc
+++ b/docs/usage/list.rst.inc
@@ -176,7 +176,7 @@ Supported fields, sorting by the same value that ``--format`` would print:
 - type, mode: the file type character resp. the full file mode string
 - user, group: the owner/group name, or the numeric id if the name is not known
 - uid, gid: the numeric owner/group id
-- size: the file size (0 for hardlink slaves, same as {size})
+- size: the file size (same as {size}, 0 for items without content)
 - mtime, ctime, atime: the timestamps (atime/ctime fall back to mtime if not stored)
 
 There is no ``birthtime`` field, as there is no {birthtime} format key either.
diff --git a/src/borg/archiver/list_cmd.py b/src/borg/archiver/list_cmd.py
index 1893b0df44..8a2cc2bf3d 100644
--- a/src/borg/archiver/list_cmd.py
+++ b/src/borg/archiver/list_cmd.py
@@ -149,7 +149,7 @@ def build_parser_list(self, subparsers, common_parser, mid_common_parser):
         - type, mode: the file type character resp. the full file mode string
         - user, group: the owner/group name, or the numeric id if the name is not known
         - uid, gid: the numeric owner/group id
-        - size: the file size (0 for hardlink slaves, same as {size})
+        - size: the file size (same as {size}, 0 for items without content)
         - mtime, ctime, atime: the timestamps (atime/ctime fall back to mtime if not stored)
 
         There is no ``birthtime`` field, as there is no {birthtime} format key either.
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index d0c423b80c..c53aa0ce9e 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -1262,7 +1262,8 @@ def calculate_num_chunks(self, item):
         return len(item.get("chunks", []))
 
     def calculate_size(self, item):
-        # note: does not support hard link slaves, they will be size 0
+        # note: 0 for items without a chunks list, e.g. directories, symlinks, devices, fifos.
+        # borg2 hard links all have their own chunks list, so they get their real size.
         return item.get_size()
 
     def calculate_fingerprint(self, item):
diff --git a/src/borg/item.pyi b/src/borg/item.pyi
index 33baf21665..1bc17a0ddf 100644
--- a/src/borg/item.pyi
+++ b/src/borg/item.pyi
@@ -184,6 +184,7 @@ class Item(PropDict):
     def hlid(self) -> bytes: ...
     @hlid.setter
     def hlid(self, val: bytes) -> None: ...
+    # legacy borg1 on-disk key, only read when transferring borg1 archives
     @property
     def hardlink_master(self) -> bool: ...
     @hardlink_master.setter
@@ -224,14 +225,7 @@ class Item(PropDict):
     def size(self) -> int: ...
     @size.setter
     def size(self, val: int) -> None: ...
-    def get_size(
-        self,
-        hardlink_masters=...,
-        memorize: bool = ...,
-        compressed: bool = ...,
-        from_chunks: bool = ...,
-        consider_ids: list[bytes] = ...,
-    ) -> int: ...
+    def get_size(self, *, memorize: bool = ..., from_chunks: bool = ..., consider_ids: list[bytes] = ...) -> int: ...
     @property
     def part(self) -> int: ...
     @part.setter
diff --git a/src/borg/item.pyx b/src/borg/item.pyx
index bec41404ae..b8388e892d 100644
--- a/src/borg/item.pyx
+++ b/src/borg/item.pyx
@@ -286,7 +286,7 @@ cdef class Item(PropDict):
     inode = PropDictProperty(int)
 
     hlid = PropDictProperty(bytes)  # hard link id: same value means same hard link.
-    hardlink_master = PropDictProperty(bool)  # legacy
+    hardlink_master = PropDictProperty(bool)  # legacy borg1 on-disk key, only read when transferring borg1 archives
 
     chunks = PropDictProperty(list, 'list')
     chunks_healthy = PropDictProperty(list, 'list')
diff --git a/src/borg/legacy/helpers.py b/src/borg/legacy/helpers.py
index f0cdebf9cf..2fd6035ecc 100644
--- a/src/borg/legacy/helpers.py
+++ b/src/borg/legacy/helpers.py
@@ -5,9 +5,18 @@ def borg1_hardlinkable(mode):
     return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode)
 
 
-def borg1_hardlink_master(item):
+def borg1_hardlink_with_content(item):
+    """Is this a borg1 hard link item that carries the content of its hard link group?
+
+    borg1 stored a group of hard links asymmetrically: the first item encountered got the
+    chunks list, every later one only got a .source back reference to that item's path.
+    Note that the 'hardlink_master' key does not tell these apart: since #7175 it is set
+    for all hardlinkable items, so the absence of .source is what identifies the item
+    that has the content.
+    """
     return item.get("hardlink_master", False) and "source" not in item and borg1_hardlinkable(item.mode)
 
 
-def borg1_hardlink_slave(item):
+def borg1_hardlink_reference(item):
+    """Is this a borg1 hard link item that only back references the item with the content?"""
     return "source" in item and borg1_hardlinkable(item.mode)
diff --git a/src/borg/legacy/upgrade.py b/src/borg/legacy/upgrade.py
index 80c3357eea..b059d4fb7f 100644
--- a/src/borg/legacy/upgrade.py
+++ b/src/borg/legacy/upgrade.py
@@ -6,7 +6,7 @@
 from ..helpers import HardLinkManager, join_cmd
 from ..item import Item
 from ..logger import create_logger
-from .helpers import borg1_hardlink_master, borg1_hardlink_slave
+from .helpers import borg1_hardlink_with_content, borg1_hardlink_reference
 
 logger = create_logger(__name__)
 
@@ -48,10 +48,10 @@ def upgrade_item(self, *, item):
             "acl_extended",
         }
 
-        if borg1_hardlink_master(item):
+        if borg1_hardlink_with_content(item):
             item.hlid = hlid = self.hlm.hardlink_id_from_path(item.path)
             self.hlm.remember(id=hlid, info=item.get("chunks"))
-        elif borg1_hardlink_slave(item):
+        elif borg1_hardlink_reference(item):
             item.hlid = hlid = self.hlm.hardlink_id_from_path(item.source)
             chunks = self.hlm.retrieve(id=hlid)
             if chunks is not None:
diff --git a/src/borg/testsuite/archiver/extract_cmd_test.py b/src/borg/testsuite/archiver/extract_cmd_test.py
index 0e1668d426..25ac8c922b 100644
--- a/src/borg/testsuite/archiver/extract_cmd_test.py
+++ b/src/borg/testsuite/archiver/extract_cmd_test.py
@@ -120,7 +120,7 @@ def test_extract_hardlinked_symlink_does_not_leak_target(archivers, request):
         archiver,
         "evil",
         [
-            # master: a symlink pointing at the external victim file
+            # first item: a symlink pointing at the external victim file
             Item(path="s", mode=stat.S_IFLNK | 0o777, target=victim_file, hlid=hlid, **attrs),
             # contentless hardlink to the same hlid
             Item(path="h", mode=stat.S_IFLNK | 0o777, hlid=hlid, **attrs),
diff --git a/src/borg/testsuite/archiver/transfer_cmd_test.py b/src/borg/testsuite/archiver/transfer_cmd_test.py
index 7db1c93987..2ba0183f08 100644
--- a/src/borg/testsuite/archiver/transfer_cmd_test.py
+++ b/src/borg/testsuite/archiver/transfer_cmd_test.py
@@ -129,7 +129,7 @@ def convert_tz(local_naive, tzoffset, tzinfo):
                 if key in e:
                     e[key] = convert_tz(e[key], repo12_tzoffset, None)
 
-            # borg 1 used hard link slaves linking back to their hard link masters.
+            # borg 1 used contentless hard link items back referencing the item with the content.
             # borg 2 uses symmetric approach: just normal items. if they are hard links,
             # each item has normal attributes, including the chunks list, size. additionally,
             # they have a hlid and same hlid means same inode / belonging to same set of hard links.
@@ -137,16 +137,16 @@ def convert_tz(local_naive, tzoffset, tzinfo):
             if hardlink:
                 hardlinks[g["path"]] = g["hlid"]
                 if e["mode"].startswith("h"):
-                    # fix expectations: borg1 signalled a hard link slave with "h"
+                    # fix expectations: borg1 signalled a contentless hard link with "h"
                     # borg2 treats all hard links symmetrically as normal files
                     e["mode"] = g["mode"][0] + e["mode"][1:]
-                    # borg1 used source/linktarget to link back to hard link master
+                    # borg1 used source/linktarget to link back to the item with the content
                     assert e["source"] != ""
                     assert e["linktarget"] != ""
                     # fix expectations: borg2 does not use source/linktarget any more for hard links
                     e["source"] = ""
                     e["linktarget"] = ""
-                    # borg 1 has size == 0 for hard link slaves, borg 2 has the real file size
+                    # borg 1 has size == 0 for contentless hard links, borg 2 has the real file size
                     assert e["size"] == 0
                     assert g["size"] >= 0
                     # fix expectation for size
diff --git a/src/borg/testsuite/legacy_helpers_test.py b/src/borg/testsuite/legacy_helpers_test.py
index c80cef4288..aa390327b9 100644
--- a/src/borg/testsuite/legacy_helpers_test.py
+++ b/src/borg/testsuite/legacy_helpers_test.py
@@ -1,9 +1,9 @@
-"""Tests for borg.legacy.helpers (borg1_hardlinkable, borg1_hardlink_master, borg1_hardlink_slave)."""
+"""Tests for borg.legacy.helpers (borg1_hardlinkable, borg1_hardlink_with_content, borg1_hardlink_reference)."""
 
 import stat
 
 from ..item import Item
-from ..legacy.helpers import borg1_hardlink_master, borg1_hardlink_slave, borg1_hardlinkable
+from ..legacy.helpers import borg1_hardlink_with_content, borg1_hardlink_reference, borg1_hardlinkable
 
 
 def _item(**kwargs):
@@ -43,38 +43,45 @@ def test_hardlinkable_socket_is_false():
     assert not borg1_hardlinkable(stat.S_IFSOCK | 0o600)
 
 
-# ── borg1_hardlink_master ─────────────────────────────────────────────────────
+# ── borg1_hardlink_with_content ───────────────────────────────────────────────
 
 
-def test_hardlink_master_true_when_all_conditions_met():
+def test_hardlink_with_content_true_when_all_conditions_met():
     item = _item(hardlink_master=True)
-    assert borg1_hardlink_master(item)
+    assert borg1_hardlink_with_content(item)
 
 
-def test_hardlink_master_false_when_flag_missing():
+def test_hardlink_with_content_false_when_flag_missing():
     item = _item()
-    assert not borg1_hardlink_master(item)
+    assert not borg1_hardlink_with_content(item)
 
 
-def test_hardlink_master_false_when_flag_is_false():
+def test_hardlink_with_content_false_when_flag_is_false():
     item = _item(hardlink_master=False)
-    assert not borg1_hardlink_master(item)
+    assert not borg1_hardlink_with_content(item)
 
 
-# ── borg1_hardlink_slave ──────────────────────────────────────────────────────
+def test_hardlink_with_content_false_when_source_present():
+    # since #7175, borg1 set the hardlink_master flag on all hardlinkable items,
+    # so only the absence of .source identifies the item that has the content.
+    item = _item(hardlink_master=True, source="dir/original")
+    assert not borg1_hardlink_with_content(item)
 
 
-def test_hardlink_slave_true_when_source_and_hardlinkable():
+# ── borg1_hardlink_reference ──────────────────────────────────────────────────
+
+
+def test_hardlink_reference_true_when_source_and_hardlinkable():
     item = _item(source="dir/original")
-    assert borg1_hardlink_slave(item)
+    assert borg1_hardlink_reference(item)
 
 
-def test_hardlink_slave_true_for_non_regular_hardlinkable_type():
+def test_hardlink_reference_true_for_non_regular_hardlinkable_type():
     # borg1_hardlinkable covers FIFO/block/char as well as regular files
     item = _item(source="dir/original", mode=stat.S_IFIFO | 0o644)
-    assert borg1_hardlink_slave(item)
+    assert borg1_hardlink_reference(item)
 
 
-def test_hardlink_slave_false_when_no_source():
+def test_hardlink_reference_false_when_no_source():
     item = _item()
-    assert not borg1_hardlink_slave(item)
+    assert not borg1_hardlink_reference(item)
diff --git a/src/borg/testsuite/legacy_upgrade_test.py b/src/borg/testsuite/legacy_upgrade_test.py
index 56903c09fb..b9884dbca8 100644
--- a/src/borg/testsuite/legacy_upgrade_test.py
+++ b/src/borg/testsuite/legacy_upgrade_test.py
@@ -81,7 +81,7 @@ def test_symlink_source_renamed_to_target():
     assert "source" not in d
 
 
-def test_hardlink_master_gets_hlid_and_strips_hardlink_master_key():
+def test_hardlink_with_content_gets_hlid_and_strips_hardlink_master_key():
     u, _, _ = _upgrader()
     item = _item(hardlink_master=True, chunks=[[CHUNK_ID, CHUNK_SIZE]])
     result = u.upgrade_item(item=item)
@@ -91,19 +91,19 @@ def test_hardlink_master_gets_hlid_and_strips_hardlink_master_key():
     assert d["chunks"] == [[CHUNK_ID, CHUNK_SIZE]]
 
 
-def test_hardlink_slave_gets_master_hlid_and_chunks():
+def test_hardlink_reference_gets_hlid_and_chunks_of_item_with_content():
     u, cache, archive = _upgrader()
-    master = _item(hardlink_master=True, chunks=[[CHUNK_ID, CHUNK_SIZE]])
-    master_result = u.upgrade_item(item=master)
+    with_content = _item(hardlink_master=True, chunks=[[CHUNK_ID, CHUNK_SIZE]])
+    with_content_result = u.upgrade_item(item=with_content)
 
-    slave = _item(path="dir/link2", source="dir/file")
-    slave_result = u.upgrade_item(item=slave)
+    reference = _item(path="dir/link2", source="dir/file")
+    reference_result = u.upgrade_item(item=reference)
 
-    d = slave_result.as_dict()
+    d = reference_result.as_dict()
     assert "hlid" in d
     assert "source" not in d
     assert d["chunks"] == [[CHUNK_ID, CHUNK_SIZE]]
-    assert slave_result.hlid == master_result.hlid
+    assert reference_result.hlid == with_content_result.hlid
     cache.reuse_chunk.assert_called_once_with(CHUNK_ID, CHUNK_SIZE, archive.stats)
 
 

From 69acada45cdc55d1e517ec9515076e93467c7c46 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Mon, 17 Aug 2026 03:26:38 +0200
Subject: [PATCH 144/144] create --sparse: fix stale help text, sparse input
 works with all chunkers

The help text still claimed sparse hole detection is "supported only by
fixed chunker", which is stale: --sparse is passed to every chunker via
get_chunker(), and ChunkerBase/FileFMAPReader implement sparse processing
generically (build the hole map via SEEK_HOLE/SEEK_DATA, seek over holes
instead of reading them, process their content as all-zero).

Verified for all chunkers (buzhash, buzhash64, fastcdc, rabin-aes,
goldilocks-aes, toeplitz-aes, fixed) by chunking a real sparse file with
sparse=True and a hole-poisoned file with an explicit fmap: content is
reproduced exactly and hole ranges are seeked over, not read.

Also move the sparse processing note in docs/internals/data-structures.rst
from the "fixed" chunker section to the chunker overview, and regenerate
the affected generated docs (docs/usage/create.rst.inc, borg-create.1).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/internals/data-structures.rst | 9 ++++-----
 docs/man/borg-create.1             | 4 ++--
 docs/usage/create.rst.inc          | 4 ++--
 src/borg/archiver/create_cmd.py    | 2 +-
 4 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst
index a90be6672d..71ec05ae5e 100644
--- a/docs/internals/data-structures.rst
+++ b/docs/internals/data-structures.rst
@@ -425,6 +425,10 @@ Borg has these chunkers (the default is "fastcdc"):
   underlying paper); about half the rabin-aes speed, mainly a comparison
   baseline.
 
+All chunkers support sparse file processing (``borg create --sparse``): hole
+ranges in the input file are then detected (via ``SEEK_HOLE``/``SEEK_DATA``)
+and seeked over instead of being read, processing their content as all-zero.
+
 For some more general usage hints see also ``--chunker-params``.
 
 "fixed" chunker
@@ -444,11 +448,6 @@ The default is not to have a differently sized header.
   bytes) recommended. E.g.: 4194304 would cut 4MiB sized chunks.
 - HEADER_SIZE: optional, defaults to 0 (no header).
 
-The fixed chunker also supports processing sparse files (reading only the ranges
-with data and seeking over the empty hole ranges).
-
-``borg create --sparse --chunker-params fixed,BLOCK_SIZE[,HEADER_SIZE]``
-
 "fastcdc" chunker
 +++++++++++++++++
 
diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1
index 7729fbbfad..d0356f3a07 100644
--- a/docs/man/borg-create.1
+++ b/docs/man/borg-create.1
@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "borg-create" "1" "2026-08-15" "" "borg backup tool"
+.TH "borg-create" "1" "2026-08-17" "" "borg backup tool"
 .SH Name
 borg-create \- Creates a new archive.
 .SH SYNOPSIS
@@ -307,7 +307,7 @@ do not read and store ACLs into archive
 do not read and store xattrs into archive
 .TP
 .B  \-\-sparse
-detect sparse holes in input (supported only by fixed chunker)
+detect sparse holes in input and seek over them instead of reading them
 .TP
 .BI \-\-files\-cache \ MODE
 operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there).
diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc
index 268b6387f1..70d9ad95a0 100644
--- a/docs/usage/create.rst.inc
+++ b/docs/usage/create.rst.inc
@@ -89,7 +89,7 @@ borg create
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |                                                       | ``--noxattrs``                                    | do not read and store xattrs into archive                                                                                                                                                                                                      |
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-    |                                                       | ``--sparse``                                      | detect sparse holes in input (supported only by fixed chunker)                                                                                                                                                                                 |
+    |                                                       | ``--sparse``                                      | detect sparse holes in input and seek over them instead of reading them                                                                                                                                                                        |
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |                                                       | ``--files-cache MODE``                            | operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there).                                                                                                              |
     +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
@@ -168,7 +168,7 @@ borg create
         --noflags                 do not read and store flags (e.g. NODUMP, IMMUTABLE) into archive
         --noacls                  do not read and store ACLs into archive
         --noxattrs                do not read and store xattrs into archive
-        --sparse                  detect sparse holes in input (supported only by fixed chunker)
+        --sparse                  detect sparse holes in input and seek over them instead of reading them
         --files-cache MODE        operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there).
         --files-changed MODE      specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there).
         --read-special            open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files.
diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py
index 5dbc2ec455..1212a739dc 100644
--- a/src/borg/archiver/create_cmd.py
+++ b/src/borg/archiver/create_cmd.py
@@ -1105,7 +1105,7 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
             "--sparse",
             dest="sparse",
             action="store_true",
-            help="detect sparse holes in input (supported only by fixed chunker)",
+            help="detect sparse holes in input and seek over them instead of reading them",
         )
         fs_group.add_argument(
             "--files-cache",