diff --git a/src/borg/archive.py b/src/borg/archive.py index 733b96122c..7bfcd2aa12 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -2024,14 +2024,18 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, return status -def tar_acl_to_borg(acl): +def tar_acl_to_borg(acl, unknown_names=None): """Convert a POSIX ACL text from a tar PAX header (SCHILY.acl.*) to borg's ACL format. Borg separates entries by newlines and appends the numeric uid/gid as a 4th field to named user/group entries (user:name:perms:uid), see acl_get on Linux and FreeBSD. star appends it too, but separates entries by commas. GNU tar separates entries by newlines and does not append the numeric id, so we look it up locally (like borg create - does), falling back to the name. + does), falling back to the name. *unknown_names* is an optional set, the (tag, name) of + the entries whose name could not be looked up get added to it. + + Raise ValueError for an entry with an unknown tag or a wrong number of fields, as the + platform code converting borg's ACL format at extract time relies on the field layout. """ entries = [] for entry in acl.replace(",", "\n").split("\n"): @@ -2039,9 +2043,20 @@ def tar_acl_to_borg(acl): if not entry: continue fields = entry.split(":") - if len(fields) == 3 and fields[1] and fields[0] in ("user", "group"): + tag = fields[0] + if tag not in ("user", "group", "mask", "other"): + raise ValueError(f"invalid POSIX ACL entry: {entry!r}") + named = tag in ("user", "group") and len(fields) > 1 and fields[1] != "" + if named and len(fields) == 3: # tag:name:perms -> append the numeric id name = fields[1] - fields.append(str(user2uid(name, name) if fields[0] == "user" else group2gid(name, name))) + numeric_id = user2uid(name) if tag == "user" else group2gid(name) + if numeric_id is None: + if unknown_names is not None: + unknown_names.add((tag, name)) + numeric_id = name + fields.append(str(numeric_id)) + elif len(fields) != (4 if named else 3): # tag:name:perms:id or tag::perms + raise ValueError(f"invalid POSIX ACL entry: {entry!r}") entries.append(":".join(fields)) return "\n".join(entries).encode("utf-8", errors="surrogateescape") @@ -2058,6 +2073,7 @@ def __init__( show_progress, log_json, file_status_printer=None, + warning, digest_algos=DIGEST_ALGOS_DEFAULT, strip_components=0, ): @@ -2068,6 +2084,8 @@ def __init__( self.show_progress = show_progress self.strip_components = strip_components self.print_file_status = file_status_printer or (lambda *args: None) + self.print_warning = warning # callback(format, *args), like Archiver.print_warning + self.unknown_acl_names = set() # (tag, name) of POSIX ACL entries we already warned about self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) self.chunker = get_chunker(*chunker_params, key=key, sparse=False) @@ -2086,7 +2104,11 @@ def create_helper(self, tarinfo, status=None, type=None): else: def s_to_ns(s): - return safe_ns(int(float(s) * 1e9)) + try: + return safe_ns(int(float(s) * 1e9)) + except (OverflowError, ValueError): + # inf or nan: the tarfile module parses a pax mtime like "1e999" or "nan" without complaint. + return 0 # if the tar has names starting with "./", normalize them like borg create also does. # ./dir/file must become dir/file in the borg archive. @@ -2121,10 +2143,25 @@ def s_to_ns(s): bkey = key.encode("utf-8", errors="surrogateescape") bvalue = value.encode("utf-8", errors="surrogateescape") xattrs[bkey] = bvalue - elif key == SCHILY_ACL_ACCESS: - item.acl_access = tar_acl_to_borg(value) - elif key == SCHILY_ACL_DEFAULT: - item.acl_default = tar_acl_to_borg(value) + elif key in (SCHILY_ACL_ACCESS, SCHILY_ACL_DEFAULT): + unknown_names = set() + try: + acl = tar_acl_to_borg(value, unknown_names=unknown_names) + except ValueError as e: + self.print_warning( + "%s: ignoring the %s PAX header: %s", remove_surrogates(item.path), key, e + ) + else: + setattr(item, "acl_access" if key == SCHILY_ACL_ACCESS else "acl_default", acl) + for tag, name in sorted(unknown_names - self.unknown_acl_names): + self.print_warning( + "%s: POSIX ACL refers to unknown %s %r, extracting it will fail unless that %s exists", + remove_surrogates(item.path), + tag, + name, + tag, + ) + self.unknown_acl_names |= unknown_names if xattrs: item.xattrs = xattrs if self.strip_components: diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index c9e12969ca..b0a6104ebf 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -112,8 +112,11 @@ def item_to_tarinfo(item, hlm, *, warning=None): return tarinfo, needs_content -def item_to_paxheaders(format, item): - """Transform (parts of) a Borg *item* into a pax_headers dict.""" +def item_to_paxheaders(format, item, *, warning=None): + """Transform (parts of) a Borg *item* into a pax_headers dict. + + *warning* is an optional callback(format, *args) for reporting metadata that can not be represented. + """ # PAX format # ---------- # When using the PAX (POSIX) format, we can support some things that aren't possible @@ -139,7 +142,14 @@ def item_to_paxheaders(format, item): for bkey, bvalue in item.xattrs.items(): # we have bytes key and bytes value, but the tarfile code # expects str key and str value. - key = SCHILY_XATTR + bkey.decode("utf-8", errors="surrogateescape") + try: + # a pax header keyword must be valid UTF-8 (the tarfile module encodes it strictly), + # only the value may be arbitrary bytes (hdrcharset=BINARY). + key = SCHILY_XATTR + bkey.decode("utf-8") + except UnicodeDecodeError: + if format != "BORG" and warning is not None: # the BORG format still has it in BORG.item.meta + warning("%s: xattr name %r is not valid UTF-8, not exported", remove_surrogates(item.path), bkey) + continue value = bvalue.decode("utf-8", errors="surrogateescape") ph[key] = value # Add POSIX access and default ACL if present @@ -499,7 +509,7 @@ def sparsify_tarinfo(item, tarinfo): tarinfo, needs_content = item_to_tarinfo(item, hlm, warning=self.print_warning) if tarinfo: if args.tar_format in ("BORG", "PAX"): - tarinfo.pax_headers = item_to_paxheaders(args.tar_format, item) + tarinfo.pax_headers = item_to_paxheaders(args.tar_format, item, warning=self.print_warning) if output_list: logging.getLogger("borg.output.list").info(remove_surrogates(orig_path)) sparse_content = sparsify_tarinfo(item, tarinfo) if args.sparse and needs_content else None @@ -566,6 +576,7 @@ def _import_tar(self, args, repository, manifest, key, cache, tarstream): show_progress=args.progress, log_json=args.log_json, file_status_printer=self.print_file_status, + warning=self.print_warning, strip_components=strip_components, ) @@ -663,6 +674,9 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): | | | no ACLs/xattrs/bsdflags | +--------------+---------------------------+----------------------------+ + The PAX and GNU formats are lossy: BSD flags, NFSv4 ACLs (FreeBSD), extended ACLs + (macOS), birthtime and content digests are not exported. Only the BORG format has them. + With ``--sparse``, files whose content contains runs of all-zero chunks are written as sparse tar members (GNU sparse format 1.0, as GNU tar creates it in POSIX mode), storing only a hole map and the non-zero data. This requires ``--tar-format BORG`` diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index ef4f23ad01..30537e1d0e 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -1,7 +1,7 @@ import os import re from datetime import UTC, datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation +from decimal import Decimal from zoneinfo import ZoneInfo @@ -107,11 +107,21 @@ def ns_to_pax_time(ns): def pax_time_to_ns(value): - """Parse a tar PAX header timestamp (decimal seconds string) into nanoseconds, return None if invalid.""" + """Parse a tar PAX header timestamp (decimal seconds string) into nanoseconds, return None if invalid. + + Out of range values are clamped like safe_ns does. + """ try: - return safe_ns(int(Decimal(value).scaleb(9))) - except (InvalidOperation, ValueError, OverflowError): + seconds = Decimal(value) + except (ArithmeticError, ValueError): # decimal.InvalidOperation is an ArithmeticError + return None + if not seconds.is_finite(): # NaN, sNaN, Infinity return None + # clamp before converting to int: a Decimal with a huge exponent (e.g. "1e999999") is cheap to + # compare, but int() would build an integer with that many digits, which takes very long. + # also, scaleb() overflows for exponents beyond the decimal context's limit. + seconds = min(max(seconds, Decimal(0)), Decimal(MAX_S)) + return int(seconds.scaleb(9)) def safe_timestamp(item_timestamp_ns): diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py index a521598fb3..3522c99e32 100644 --- a/src/borg/testsuite/archive_test.py +++ b/src/borg/testsuite/archive_test.py @@ -876,6 +876,32 @@ def test_tar_acl_to_borg(acl, expected): assert tar_acl_to_borg(acl) == expected.encode() +@pytest.mark.parametrize( + "acl", + [ + "user:foo", # too few fields: the Linux acl_set code would crash on this at extract time + "user:foo:rw-:0:x", # too many fields + "user::rw-:0", # an id for an unnamed entry + "mask:x:rw-:0", + "bogus::rw-", # unknown tag + "rw-", + "user::rw-\ngroup::r--\nother::r--\nuser:foo", # one invalid entry spoils it all + ], +) +def test_tar_acl_to_borg_invalid(acl): + with pytest.raises(ValueError, match="invalid POSIX ACL entry"): + tar_acl_to_borg(acl) + + +@pytest.mark.skipif(is_win32, reason="no name lookups on Windows") +def test_tar_acl_to_borg_unknown_names(): + unknown_names = set() + acl = "user::rw-\nuser:nosuchuser-borgtest:rw-\ngroup:nosuchgroup-borgtest:r--\ngroup:root:r--:0\nother::r--" + tar_acl_to_borg(acl, unknown_names=unknown_names) + # group:root has its id already, so no lookup is needed for it. + assert unknown_names == {("user", "nosuchuser-borgtest"), ("group", "nosuchgroup-borgtest")} + + def test_reject_non_sanitized_item(): for path in rejected_dotdot_paths: with pytest.raises(ValueError, match="unexpected '..' element in path"): diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py index 97573567ea..28a38f9421 100644 --- a/src/borg/testsuite/archiver/tar_cmds_test.py +++ b/src/borg/testsuite/archiver/tar_cmds_test.py @@ -15,12 +15,14 @@ from ...archiver.tar_cmds import chunks_to_sparse_info, gnu_sparse_10_map, SparseTarInfo from ...constants import * # NOQA from ...helpers import Error +from ...helpers.time import MAX_NS from ...item import ChunkListEntry from .. import changedir from . import assert_dirs_equal, _extract_hardlinks_setup, cmd, open_archive, requires_hardlinks, RK_ENCRYPTION from . import create_test_files, create_regular_file from . import generate_archiver_tests from ...platform import acl_get, acl_set +from ...platformflags import is_win32 from ..platform.platform_test import skipif_not_linux, skipif_acls_not_working pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA @@ -599,6 +601,35 @@ def test_roundtrip_pax_xattrs(archivers, request): assert xa_value_extracted == xa_value +def test_export_tar_pax_xattr_name_not_utf8(archivers, request): + """export-tar --tar-format=PAX skips xattrs with a non-UTF-8 name (PAX keywords must be UTF-8) with a warning.""" + archiver = request.getfixturevalue(archivers) + if not xattr.is_enabled(archiver.input_path): + pytest.skip("xattrs not supported") + create_regular_file(archiver.input_path, "file") + original_path = os.path.join(archiver.input_path, "file").encode() + xa_key, xa_value = b"user.not-utf8-\xff", b"value" + try: + xattr.setxattr(original_path, xa_key, xa_value) + except OSError as e: + pytest.skip(f"can not set an xattr with a non-UTF-8 name here: {e}") + xattr.setxattr(original_path, b"user.utf8", b"other") + cmd(archiver, "repo-create", "--encryption=authenticated-sha256") + cmd(archiver, "create", "src", "input") + output = cmd(archiver, "export-tar", "src", "pax.tar", "--tar-format=PAX", exit_code=EXIT_WARNING) + assert "is not valid UTF-8, not exported" in output + with tarfile.open("pax.tar") as tar: + ph = tar.getmember("input/file").pax_headers + assert ph[SCHILY_XATTR + "user.utf8"] == "other" + assert not any(key.startswith(SCHILY_XATTR + "user.not-utf8") for key in ph) + # the BORG format has all item metadata in BORG.item.meta, so nothing is lost there (and no warning given). + cmd(archiver, "export-tar", "src", "borg.tar", "--tar-format=BORG") + cmd(archiver, "import-tar", "dst", "borg.tar") + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + assert xattr.getxattr(os.path.abspath("input/file").encode(), xa_key) == xa_value + + def test_roundtrip_pax_timestamps(archivers, request): """export-tar --tar-format=PAX and import-tar keep the timestamps with exact ns precision.""" archiver = request.getfixturevalue(archivers) @@ -606,7 +637,7 @@ def test_roundtrip_pax_timestamps(archivers, request): mtime_ns = 1700000000_987654321 # float seconds would round this to ~240 ns os.utime(os.path.join(archiver.input_path, "file"), ns=(mtime_ns, mtime_ns)) cmd(archiver, "repo-create", "--encryption=authenticated-sha256") - cmd(archiver, "create", "src", "input") + cmd(archiver, "create", "--atime", "src", "input") cmd(archiver, "export-tar", "src", "pax.tar", "--tar-format=PAX") cmd(archiver, "import-tar", "dst", "pax.tar") @@ -619,8 +650,12 @@ def get_times(archive): src_times, dst_times = get_times("src"), get_times("dst") assert dst_times == src_times with tarfile.open("pax.tar") as tar: - pax_mtime = tar.getmember("input/file").pax_headers["mtime"] - assert pax_mtime == f"{src_times['mtime'] // 10**9}.{src_times['mtime'] % 10**9:09d}" + ph = tar.getmember("input/file").pax_headers + for name, ns in src_times.items(): + if ns is None: # e.g. no ctime on Windows + assert name not in ph + else: + assert ph[name] == f"{ns // 10**9}.{ns % 10**9:09d}" def test_import_tar_invalid_pax_timestamp(archivers, request): @@ -639,6 +674,62 @@ def test_import_tar_invalid_pax_timestamp(archivers, request): assert item.ctime == 1700000000_500000000 +def test_import_tar_inf_nan_pax_mtime(archivers, request): + """import-tar survives pax mtime headers that the tarfile module parses into float inf / nan.""" + archiver = request.getfixturevalue(archivers) + with tarfile.open("input.tar", "w", format=tarfile.PAX_FORMAT) as tar: + for name, mtime in (("inf", "1e999"), ("nan", "nan")): + tarinfo = tarfile.TarInfo(name) + tarinfo.pax_headers = {"mtime": mtime} + tar.addfile(tarinfo, io.BytesIO()) + cmd(archiver, "repo-create", "--encryption=authenticated-sha256") + cmd(archiver, "import-tar", "dst", "input.tar") + archive_obj, repository = open_archive(archiver.repository_path, "dst") + with repository: + mtimes = {item.path: item.mtime for item in archive_obj.iter_items()} + assert mtimes == {"inf": MAX_NS, "nan": 0} # "1e999" gets clamped, "nan" is unparsable + + +def test_import_tar_invalid_pax_acl(archivers, request): + """import-tar ignores a malformed SCHILY.acl.* PAX header with a warning (stored, it would crash extract).""" + archiver = request.getfixturevalue(archivers) + with tarfile.open("input.tar", "w", format=tarfile.PAX_FORMAT) as tar: + tarinfo = tarfile.TarInfo("file") + tarinfo.pax_headers = {SCHILY_ACL_ACCESS: "user::rw-\ngroup::r--\nother::r--\nuser:foo"} + tar.addfile(tarinfo, io.BytesIO()) + cmd(archiver, "repo-create", "--encryption=authenticated-sha256") + output = cmd(archiver, "import-tar", "dst", "input.tar", exit_code=EXIT_WARNING) + assert "file: ignoring the SCHILY.acl.access PAX header: invalid POSIX ACL entry: 'user:foo'" in output + archive_obj, repository = open_archive(archiver.repository_path, "dst") + with repository: + item = next(archive_obj.iter_items()) + assert "acl_access" not in item + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + + +@pytest.mark.skipif(is_win32, reason="no name lookups on Windows") +def test_import_tar_acl_unknown_name(archivers, request): + """import-tar warns once per user/group name in POSIX ACL PAX headers that is unknown locally.""" + archiver = request.getfixturevalue(archivers) + acl = "user::rw-\nuser:nosuchuser-borgtest:rw-\ngroup::r--\nmask::rw-\nother::r--\n" + with tarfile.open("input.tar", "w", format=tarfile.PAX_FORMAT) as tar: + for name in ("file1", "file2"): + tarinfo = tarfile.TarInfo(name) + tarinfo.pax_headers = {SCHILY_ACL_ACCESS: acl} + tar.addfile(tarinfo, io.BytesIO()) + cmd(archiver, "repo-create", "--encryption=authenticated-sha256") + output = cmd(archiver, "import-tar", "dst", "input.tar", exit_code=EXIT_WARNING) + assert "file1: POSIX ACL refers to unknown user 'nosuchuser-borgtest'" in output + assert output.count("unknown user 'nosuchuser-borgtest'") == 1 + # the ACL is stored nevertheless, with the name in place of the numeric id (like borg create does) + archive_obj, repository = open_archive(archiver.repository_path, "dst") + with repository: + acls = {item.path: item.acl_access for item in archive_obj.iter_items()} + expected = b"user::rw-\nuser:nosuchuser-borgtest:rw-:nosuchuser-borgtest\ngroup::r--\nmask::rw-\nother::r--" + assert acls == {"file1": expected, "file2": expected} + + def _sparse_entries(sizes): return [ChunkListEntry(id=bytes([i]) * 32, size=size) for i, size in enumerate(sizes)] diff --git a/src/borg/testsuite/helpers/time_test.py b/src/borg/testsuite/helpers/time_test.py index 44ba2da92e..b5f1245067 100644 --- a/src/borg/testsuite/helpers/time_test.py +++ b/src/borg/testsuite/helpers/time_test.py @@ -3,7 +3,7 @@ from ...helpers.time import safe_ns, safe_s, safe_timestamp, SUPPORT_32BIT_PLATFORMS, calculate_relative_offset from ...helpers.time import format_time, format_time_ns, OutputTimestamp -from ...helpers.time import ns_to_pax_time, pax_time_to_ns, parse_local_timestamp +from ...helpers.time import ns_to_pax_time, pax_time_to_ns, parse_local_timestamp, MAX_NS from ...helpers import time as time_module @@ -99,10 +99,15 @@ def test_ns_to_pax_time(ns, expected): ("1700000000.987654321", 1700000000_987654321), ("1700000000", 1700000000_000000000), ("1700000000.5", 1700000000_500000000), - ("-1.5", 0), # safe_ns clamps + ("-1.5", 0), # clamped like safe_ns does + ("1e30", MAX_NS), + ("1e999999", MAX_NS), # must not build a 1-million-digit integer + ("-1e999999", 0), + ("1e3000000", MAX_NS), # beyond the decimal context's exponent limit ("", None), ("garbage", None), ("NaN", None), + ("sNaN", None), ("Infinity", None), ], )