diff --git a/src/borg/archive.py b/src/borg/archive.py index 1e907906a3..e5884efc62 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -42,7 +42,7 @@ from .helpers import safe_encode, make_path_safe, remove_surrogates, text_to_json, join_cmd, remove_dotdot_prefixes from .helpers import StableDict from .helpers import bin_to_hex -from .helpers import safe_ns +from .helpers import safe_ns, pax_time_to_ns from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi, get_progress_dt from .helpers import os_open, flags_normal, flags_dir, O_, SpecialFileReader from .helpers import MAP_DATA, MAP_ZERO, MAP_SAME, input_map_check_size @@ -2001,6 +2001,28 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, return status +def tar_acl_to_borg(acl): + """Convert a POSIX ACL text from a tar PAX header (SCHILY.acl.*) to borg's ACL format. + + Borg separates entries by newlines and appends the numeric uid/gid as a 4th field to + named user/group entries (user:name:perms:uid), see acl_get on Linux and FreeBSD. + star appends it too, but separates entries by commas. GNU tar separates entries by + newlines and does not append the numeric id, so we look it up locally (like borg create + does), falling back to the name. + """ + entries = [] + for entry in acl.replace(",", "\n").split("\n"): + entry = entry.split("#", 1)[0].strip() # remove comments + if not entry: + continue + fields = entry.split(":") + if len(fields) == 3 and fields[1] and fields[0] in ("user", "group"): + name = fields[1] + fields.append(str(user2uid(name, name) if fields[0] == "user" else group2gid(name, name))) + entries.append(":".join(fields)) + return "\n".join(entries).encode("utf-8", errors="surrogateescape") + + class TarfileObjectProcessors: def __init__( self, @@ -2058,27 +2080,28 @@ def s_to_ns(s): if tarinfo.gname: item.group = tarinfo.gname if ph: - # note: for mtime this is a bit redundant as it is already done by tarfile module, - # but we just do it in our way to be consistent for sure. + # the tarfile module only gives us float timestamps, parse the original strings for full precision. for name in "atime", "ctime", "mtime": if name in ph: - ns = s_to_ns(ph[name]) - setattr(item, name, ns) + ns = pax_time_to_ns(ph[name]) + if ns is not None: + setattr(item, name, ns) xattrs = StableDict() for key, value in ph.items(): if key.startswith(SCHILY_XATTR): key = key.removeprefix(SCHILY_XATTR) + if key.startswith("system.posix_acl_"): + # like borg create, we store the POSIX ACLs separately, not as xattrs. + continue # the tarfile code gives us str keys and str values, # but we need bytes keys and bytes values. bkey = key.encode("utf-8", errors="surrogateescape") bvalue = value.encode("utf-8", errors="surrogateescape") xattrs[bkey] = bvalue elif key == SCHILY_ACL_ACCESS: - # Process POSIX access ACL - item.acl_access = value.encode("utf-8", errors="surrogateescape") + item.acl_access = tar_acl_to_borg(value) elif key == SCHILY_ACL_DEFAULT: - # Process POSIX default ACL - item.acl_default = value.encode("utf-8", errors="surrogateescape") + item.acl_default = tar_acl_to_borg(value) if xattrs: item.xattrs = xattrs if self.strip_components: diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index daf73e86ab..c9e12969ca 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -27,7 +27,7 @@ from ..helpers import FilesystemPathSpec from ..helpers import make_path_safe from ..helpers import remove_surrogates -from ..helpers import timestamp, archive_ts_now +from ..helpers import timestamp, archive_ts_now, ns_to_pax_time from ..helpers import basic_json_data, json_print from ..helpers import log_multi from ..helpers.argparsing import ArgumentParser @@ -130,11 +130,11 @@ def item_to_paxheaders(format, item): # ph = {} # note: for mtime this is a bit redundant as it is already done by tarfile module, - # but we just do it in our way to be consistent for sure. + # but it only has a float, so we do it in our way to have exact ns precision. for name in "atime", "ctime", "mtime": if hasattr(item, name): ns = getattr(item, name) - ph[name] = str(ns / 1e9) + ph[name] = ns_to_pax_time(ns) if hasattr(item, "xattrs"): for bkey, bvalue in item.xattrs.items(): # we have bytes key and bytes value, but the tarfile code @@ -657,7 +657,7 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): | BORG | BORG specific, like PAX | all as supported by borg | +--------------+---------------------------+----------------------------+ | PAX | POSIX.1-2001 (pax) format | GNU + atime/ctime/mtime ns | - | | | + xattrs | + | | | + xattrs, POSIX ACLs | +--------------+---------------------------+----------------------------+ | GNU | GNU tar format | mtime s, no atime/ctime, | | | | no ACLs/xattrs/bsdflags | diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index a5656ea7fa..9587a715e1 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -67,6 +67,7 @@ from .progress import get_progress_dt, progress_wanted from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now +from .time import ns_to_pax_time, pax_time_to_ns from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH from .msgpack import is_slow_msgpack, is_supported_msgpack, get_limited_unpacker diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index a3d60600db..b0610213fb 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -1,6 +1,7 @@ import os import re from datetime import UTC, datetime, timedelta, timezone +from decimal import Decimal, InvalidOperation from zoneinfo import ZoneInfo @@ -94,6 +95,21 @@ def safe_ns(ts): return MAX_NS +def ns_to_pax_time(ns): + """Format a nanoseconds timestamp as an exact decimal seconds string for a tar PAX header.""" + sign = "-" if ns < 0 else "" + s, ns = divmod(abs(ns), 1000000000) + return f"{sign}{s}.{ns:09d}" + + +def pax_time_to_ns(value): + """Parse a tar PAX header timestamp (decimal seconds string) into nanoseconds, return None if invalid.""" + try: + return safe_ns(int(Decimal(value).scaleb(9))) + except (InvalidOperation, ValueError, OverflowError): + return None + + def safe_timestamp(item_timestamp_ns): t_ns = safe_ns(item_timestamp_ns) return utcfromtimestampns(t_ns) # return tz-aware utc datetime obj diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py index e8c0f67f5f..23069054c2 100644 --- a/src/borg/testsuite/archive_test.py +++ b/src/borg/testsuite/archive_test.py @@ -15,7 +15,7 @@ from ..archive import ITEM_KEYS, Statistics from ..archive import zero_chunk_flags, zero_chunk_id, zero_chunk_ids from ..archive import BackupOSError, BackupRaceConditionError, backup_io, backup_io_iter, get_item_uid_gid -from ..archive import stat_update_check +from ..archive import stat_update_check, tar_acl_to_borg from ..helpers import msgpack, StableDict from ..repoobj import RepoObj from ..item import Item, ArchiveItem @@ -802,6 +802,46 @@ def test_get_item_uid_gid(): assert gid == 16 +@pytest.mark.parametrize( + "acl, expected", + [ + # GNU tar: newline separated, no numeric id for named entries + ( + "user::rw-\nuser:{user}:rw-\ngroup::r--\nmask::rw-\nother::r--\n", + "user::rw-\nuser:{user}:rw-:{uid}\ngroup::r--\nmask::rw-\nother::r--", + ), + # star: comma separated, numeric id appended + ( + "user::rw-,user:root:rw-:0,group::r--,mask::rw-,other::r--", + "user::rw-\nuser:root:rw-:0\ngroup::r--\nmask::rw-\nother::r--", + ), + # borg export-tar: newline separated, numeric id appended + ( + "user::rw-\nuser:root:rw-:0\ngroup::r--\nmask::rw-\nother::r--", + "user::rw-\nuser:root:rw-:0\ngroup::r--\nmask::rw-\nother::r--", + ), + # unknown names fall back to the name (Windows maps every name to 0) + pytest.param( + "group:nosuchgroup-borgtest:r--", + "group:nosuchgroup-borgtest:r--:nosuchgroup-borgtest", + marks=pytest.mark.skipif(is_win32, reason="no name lookups on Windows"), + ), + # comments get removed + ("user:{user}:r--\t#effective:r--\n", "user:{user}:r--:{uid}"), + ("", ""), + ], +) +def test_tar_acl_to_borg(acl, expected): + # the name lookups need an existing user, e.g. Haiku has no "root" user. + try: + uid = os.getuid() # UNIX only + except AttributeError: + uid = 0 + user = uid2user(uid) + acl, expected = acl.format(user=user, uid=uid), expected.format(user=user, uid=uid) + assert tar_acl_to_borg(acl) == expected.encode() + + def test_reject_non_sanitized_item(): for path in rejected_dotdot_paths: with pytest.raises(ValueError, match="unexpected '..' element in path"): diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py index ad0fd216ee..97573567ea 100644 --- a/src/borg/testsuite/archiver/tar_cmds_test.py +++ b/src/borg/testsuite/archiver/tar_cmds_test.py @@ -599,6 +599,46 @@ def test_roundtrip_pax_xattrs(archivers, request): assert xa_value_extracted == xa_value +def test_roundtrip_pax_timestamps(archivers, request): + """export-tar --tar-format=PAX and import-tar keep the timestamps with exact ns precision.""" + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file") + mtime_ns = 1700000000_987654321 # float seconds would round this to ~240 ns + os.utime(os.path.join(archiver.input_path, "file"), ns=(mtime_ns, mtime_ns)) + cmd(archiver, "repo-create", "--encryption=authenticated-sha256") + cmd(archiver, "create", "src", "input") + cmd(archiver, "export-tar", "src", "pax.tar", "--tar-format=PAX") + cmd(archiver, "import-tar", "dst", "pax.tar") + + def get_times(archive): + archive_obj, repository = open_archive(archiver.repository_path, archive) + with repository: + item = next(item for item in archive_obj.iter_items() if item.path == "input/file") + return {name: item.get(name) for name in ("atime", "ctime", "mtime")} + + src_times, dst_times = get_times("src"), get_times("dst") + assert dst_times == src_times + with tarfile.open("pax.tar") as tar: + pax_mtime = tar.getmember("input/file").pax_headers["mtime"] + assert pax_mtime == f"{src_times['mtime'] // 10**9}.{src_times['mtime'] % 10**9:09d}" + + +def test_import_tar_invalid_pax_timestamp(archivers, request): + """import-tar ignores invalid PAX timestamps (tarfile ignores them, too).""" + archiver = request.getfixturevalue(archivers) + with tarfile.open("input.tar", "w", format=tarfile.PAX_FORMAT) as tar: + tarinfo = tarfile.TarInfo("file") + tarinfo.pax_headers = {"atime": "garbage", "ctime": "1700000000.5"} + tar.addfile(tarinfo, io.BytesIO()) + cmd(archiver, "repo-create", "--encryption=authenticated-sha256") + cmd(archiver, "import-tar", "dst", "input.tar") + archive_obj, repository = open_archive(archiver.repository_path, "dst") + with repository: + item = next(archive_obj.iter_items()) + assert "atime" not in item + assert item.ctime == 1700000000_500000000 + + def _sparse_entries(sizes): return [ChunkListEntry(id=bytes([i]) * 32, size=size) for i, size in enumerate(sizes)] @@ -937,3 +977,44 @@ def set_acl(path, access=None, default=None): assert "acl_default" in extracted_dir_acl assert extracted_dir_acl["acl_default"] == dir_acl["acl_default"] assert b"user:root:r--" in dir_acl["acl_default"] + + +@skipif_not_linux +@skipif_acls_not_working +def test_import_tar_gnu_tar_acls(archivers, request): + """Test import-tar with POSIX ACLs in PAX headers like GNU tar writes them (no numeric ids).""" + archiver = request.getfixturevalue(archivers) + + def get_acl(path): + item = {} + acl_get(path, item, os.stat(path)) + return item + + # GNU tar --format=posix --acls writes these headers, see also tar_acl_to_borg. + with tarfile.open("gnu.tar", "w", format=tarfile.PAX_FORMAT) as tar: + tarinfo = tarfile.TarInfo("dir") + tarinfo.type, tarinfo.mode = tarfile.DIRTYPE, 0o755 + tarinfo.pax_headers = { + "SCHILY.acl.access": "user::rwx\ngroup::r-x\nother::r-x\n", + "SCHILY.acl.default": "user::rwx\nuser:root:r-x\ngroup::r-x\nmask::r-x\nother::r-x\n", + } + tar.addfile(tarinfo) + tarinfo = tarfile.TarInfo("dir/file") + tarinfo.mode = 0o644 + tarinfo.pax_headers = { + "SCHILY.acl.access": "user::rw-\nuser:root:rw-\ngroup::r--\nmask::rw-\nother::r--\n", + # GNU tar --xattrs-include='*' also stores the ACLs as raw xattrs, these must be ignored. + "SCHILY.xattr.system.posix_acl_access": "not a valid binary ACL", + } + tar.addfile(tarinfo, io.BytesIO()) + + cmd(archiver, "repo-create", "--encryption=authenticated-sha256") + cmd(archiver, "import-tar", "dst", "gnu.tar") + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + file_acl = get_acl(os.path.abspath("dir/file")) + dir_acl = get_acl(os.path.abspath("dir")) + if not file_acl.get("acl_access") or not dir_acl.get("acl_default"): + pytest.skip("ACLs not supported or not working correctly") + assert b"user:root:rw-:0" in file_acl["acl_access"] + assert b"user:root:r-x:0" in dir_acl["acl_default"] diff --git a/src/borg/testsuite/helpers/time_test.py b/src/borg/testsuite/helpers/time_test.py index 2975bd23ac..8b929213d2 100644 --- a/src/borg/testsuite/helpers/time_test.py +++ b/src/borg/testsuite/helpers/time_test.py @@ -3,6 +3,7 @@ from ...helpers.time import safe_ns, safe_s, safe_timestamp, SUPPORT_32BIT_PLATFORMS, calculate_relative_offset from ...helpers.time import format_time, format_time_ns, OutputTimestamp +from ...helpers.time import ns_to_pax_time, pax_time_to_ns def utcfromtimestamp(timestamp): @@ -75,3 +76,34 @@ def test_output_timestamp_without_ns_isoformat(): ns = 1000000000_000123_456 ots = OutputTimestamp(safe_timestamp(ns)) # no ns given assert ots.isoformat() == safe_timestamp(ns).astimezone().isoformat(timespec="microseconds") + + +@pytest.mark.parametrize( + "ns, expected", + [ + (0, "0.000000000"), + (1, "0.000000001"), + (1700000000_987654321, "1700000000.987654321"), + (-1, "-0.000000001"), + (-1500000000, "-1.500000000"), + ], +) +def test_ns_to_pax_time(ns, expected): + assert ns_to_pax_time(ns) == expected + + +@pytest.mark.parametrize( + "value, expected", + [ + ("1700000000.987654321", 1700000000_987654321), + ("1700000000", 1700000000_000000000), + ("1700000000.5", 1700000000_500000000), + ("-1.5", 0), # safe_ns clamps + ("", None), + ("garbage", None), + ("NaN", None), + ("Infinity", None), + ], +) +def test_pax_time_to_ns(value, expected): + assert pax_time_to_ns(value) == expected