diff --git a/docs/man/borg-known-repos.1 b/docs/man/borg-known-repos.1 new file mode 100644 index 0000000000..fa5f0ecc6e --- /dev/null +++ b/docs/man/borg-known-repos.1 @@ -0,0 +1,92 @@ +.\" Man page generated from reStructuredText +.\" by the Docutils 0.22.4 manpage writer. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "borg-known-repos" "1" "2026-10-13" "" "borg backup tool" +.SH Name +borg-known-repos \- List repositories Borg has local state for (security info and/or cache). +.SH SYNOPSIS +.sp +borg [common options] known\-repos [options] +.SH DESCRIPTION +.sp +Lists the repositories this local Borg installation currently has state for \- +security info (key type, last known location), a local repository cache, and/or key +files \- without accessing any repository itself (\-r/\-\-repo has no effect). \(dqLayout\(dq +is which state was found in (not the on\-disk repository format, see \(dqborg repo\-info\(dq +for that) \- a security\-only repo gets \(dqunknown\(dq when $BORG_SECURITY_DIR makes the +borg2 and legacy security roots coincide; \(dqLast written\(dq is the cache config\(aqs mtime, +a best\-effort last\-used proxy, not an exact timestamp and not a staleness check on its +own; a mismatch between a legacy repo\(aqs known location and its cache\(aqs last location +is flagged; a key file\(aqs presence does not by itself mean the repository is still +openable. Use \-\-json for a machine\-readable version. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS options +.INDENT 0.0 +.TP +.B \-\-json +format output as JSON +.UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +$ borg known\-repos +Repository ID Layout Cache Security Key files Last written Location +401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb borg2 180 B aes256\-ocb, sha256 \- 2026\-10\-12T17:15:19 /path/to/repo + +$ borg known\-repos \-\-json +{ + \(dqrepositories\(dq: [ + { + \(dqcache_config_mtime\(dq: \(dq2026\-10\-12T17:15:19.785785+09:00\(dq, + \(dqcache_size\(dq: 180, + \(dqhas_cache\(dq: true, + \(dqhas_security_info\(dq: true, + \(dqid\(dq: \(dq401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb\(dq, + \(dqkey_file_count\(dq: 0, + \(dqkey_type\(dq: \(dqaes256\-ocb, sha256\(dq, + \(dqlayout\(dq: \(dqborg2\(dq, + \(dqlocation\(dq: \(dq/path/to/repo\(dq, + \(dqlocation_mismatch\(dq: false + } + ] +} +.EE +.UNINDENT +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH Author +The Borg Collective +.\" End of generated man page. diff --git a/docs/usage.rst b/docs/usage.rst index 02a74f95e2..ffc29721a4 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -44,6 +44,7 @@ Usage usage/repo-delete usage/serve usage/version + usage/known-repos usage/compact usage/lock usage/key diff --git a/docs/usage/known-repos.rst b/docs/usage/known-repos.rst new file mode 100644 index 0000000000..c8f518005b --- /dev/null +++ b/docs/usage/known-repos.rst @@ -0,0 +1,27 @@ +.. include:: known-repos.rst.inc + +Examples +~~~~~~~~ +:: + + $ borg known-repos + Repository ID Layout Cache Security Key files Last written Location + 401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb borg2 180 B aes256-ocb, sha256 - 2026-10-12T17:15:19 /path/to/repo + + $ borg known-repos --json + { + "repositories": [ + { + "cache_config_mtime": "2026-10-12T17:15:19.785785+09:00", + "cache_size": 180, + "has_cache": true, + "has_security_info": true, + "id": "401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb", + "key_file_count": 0, + "key_type": "aes256-ocb, sha256", + "layout": "borg2", + "location": "/path/to/repo", + "location_mismatch": false + } + ] + } diff --git a/docs/usage/known-repos.rst.inc b/docs/usage/known-repos.rst.inc new file mode 100644 index 0000000000..9ed08ffcc7 --- /dev/null +++ b/docs/usage/known-repos.rst.inc @@ -0,0 +1,56 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_known-repos: + +borg known-repos +---------------- +.. code-block:: none + + borg [common options] known-repos [options] + +.. only:: html + + .. class:: borg-options-table + + +-------------------------------------------------------+------------+-----------------------+ + | **options** | + +-------------------------------------------------------+------------+-----------------------+ + | | ``--json`` | format output as JSON | + +-------------------------------------------------------+------------+-----------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+------------+-----------------------+ + + .. raw:: html + + + +.. only:: latex + + + + options + --json format output as JSON + + + :ref:`common_options` + | + +Description +~~~~~~~~~~~ + +Lists the repositories this local Borg installation currently has state for - +security info (key type, last known location), a local repository cache, and/or key +files - without accessing any repository itself (-r/--repo has no effect). "Layout" +is which state was found in (not the on-disk repository format, see "borg repo-info" +for that) - a security-only repo gets "unknown" when $BORG_SECURITY_DIR makes the +borg2 and legacy security roots coincide; "Last written" is the cache config's mtime, +a best-effort last-used proxy, not an exact timestamp and not a staleness check on its +own; a mismatch between a legacy repo's known location and its cache's last location +is flagged; a key file's presence does not by itself mean the repository is still +openable. Use --json for a machine-readable version. \ No newline at end of file diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index caa320611a..4bbd796da6 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -79,6 +79,7 @@ from .help_cmd import HelpMixIn from .info_cmd import InfoMixIn from .key_cmds import KeysMixIn +from .known_repos_cmd import KnownReposMixIn from .list_cmd import ListMixIn from .lock_cmds import LocksMixIn from .mount_cmds import MountMixIn @@ -116,6 +117,7 @@ class Archiver( HelpMixIn, InfoMixIn, KeysMixIn, + KnownReposMixIn, ListMixIn, LocksMixIn, MountMixIn, @@ -310,6 +312,7 @@ def build_parser(self): self.build_parser_help(subparsers, common_parser, mid_common_parser, parser) self.build_parser_info(subparsers, common_parser, mid_common_parser) self.build_parser_keys(subparsers, common_parser, mid_common_parser) + self.build_parser_known_repos(subparsers, common_parser, mid_common_parser) self.build_parser_list(subparsers, common_parser, mid_common_parser) self.build_parser_locks(subparsers, common_parser, mid_common_parser) self.build_parser_mount_umount(subparsers, common_parser, mid_common_parser) diff --git a/src/borg/archiver/known_repos_cmd.py b/src/borg/archiver/known_repos_cmd.py new file mode 100644 index 0000000000..9ea76835e6 --- /dev/null +++ b/src/borg/archiver/known_repos_cmd.py @@ -0,0 +1,367 @@ +import os +import re +import stat +from configparser import ConfigParser, Error as ConfigParserError +from pathlib import Path + +from ..crypto.key import LEGACY_KEY_TYPES, AVAILABLE_KEY_TYPES, KEYFILE_ID +from ..helpers import format_file_size, json_print, get_security_dir, get_cache_dir, get_keys_dir +from ..helpers import safe_timestamp, OutputTimestamp, normalize_local_path +from ..helpers.argparsing import ArgumentParser +from ..legacy.fs import get_cache_dir as get_cache_dir_legacy +from ..legacy.fs import get_security_dir as get_security_dir_legacy +from ..legacy.fs import get_keys_dir as get_keys_dir_legacy + +from ..logger import create_logger + +logger = create_logger() + +REPO_ID_RE = re.compile(r"^[0-9a-f]{64}\Z") + + +def _printable(s): + return "".join(c if c.isprintable() else repr(c)[1:-1] for c in s) + + +def _build_key_type_labels(): + labels = {} + for key_cls in LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES: + is_legacy = key_cls in LEGACY_KEY_TYPES + label = key_cls.ENC_NAME + if not key_cls.IDHASH_IN_ENC_NAME: + label += f", {key_cls.IDHASH_NAME}" + if is_legacy: + label += " (legacy)" + for type_byte in key_cls.TYPES_ACCEPTABLE: + labels[str(type_byte)] = label + return labels + + +KEY_TYPE_LABELS = _build_key_type_labels() + + +class _Scanner: + def __init__(self): + self.incomplete = False + + def warn(self, msg, *args): + self.incomplete = True + logger.warning(msg, *(_printable(str(a)) if isinstance(a, (str, OSError)) else a for a in args)) + + def _list_dir(self, root): + """List root's direct children, or [] if it does not exist - an OSError beyond that + (e.g. a configured root that is not actually a directory, EACCES, EIO) is warned on + and also counted via warn(). + """ + try: + return list(root.iterdir()) + except FileNotFoundError: + return [] + except OSError as exc: + self.warn("known-repos: could not list %s, results may be incomplete: %s", root, exc) + return [] + + def repo_id_dirs(self, root): + for entry in self._list_dir(root): + if REPO_ID_RE.match(entry.name) and self._is_dir(entry): + yield entry.name, entry + + def _stat(self, path): + """Symlink-following stat(), or None if the path does not exist - an OSError beyond + that (e.g. EACCES, EIO) is warned on and also counted via warn(). + """ + try: + return path.stat() + except FileNotFoundError: + return None + except OSError as exc: + self.warn("known-repos: could not check %s: %s", path, exc) + return None + + def _is_dir(self, path): + st = self._stat(path) + return st is not None and stat.S_ISDIR(st.st_mode) + + def is_real_file(self, path): + st = self._stat(path) + return st is not None and stat.S_ISREG(st.st_mode) + + def marker_exists(self, path): + """True if the marker is confirmed present, False otherwise - including on a + filesystem error, which is already warned on and counted via self._stat(). + """ + return self._stat(path) is not None + + def read_text(self, path): + try: + return path.read_text() + except FileNotFoundError: + return None + except OSError as exc: + self.warn("known-repos: could not read %s: %s", path, exc) + return None + except UnicodeDecodeError as exc: + self.warn("known-repos: %s is not valid UTF-8, ignoring its content: %s", path, exc) + return None + + def read_first_line(self, path, max_size=128): + """Read only the first max_size bytes of a (possibly large) file and return the line + up to the first newline within them - used for key files, so most of a key file's + content (the encrypted key material after the header) is never read. The header + itself ("BORG_KEY ", 73 bytes) always fits within max_size, so this never + needs to read past it in practice. + """ + try: + with open(path, "rb") as fd: + chunk = fd.read(max_size) + except FileNotFoundError: + return None + except OSError as exc: + self.warn("known-repos: could not read %s: %s", path, exc) + return None + line = chunk.split(b"\n", 1)[0] + try: + return line.decode("utf-8") + except UnicodeDecodeError: + return None + + def plain_files(self, root): + """Yield (name, path) for every regular file directly under root.""" + for entry in self._list_dir(root): + if self.is_real_file(entry): + yield entry.name, entry + + def mtime(self, path): + st = self._stat(path) + return OutputTimestamp(safe_timestamp(st.st_mtime_ns)) if st is not None else None + + def dir_size(self, path): + total = 0 + for dirpath, dirnames, filenames in os.walk(path, onerror=self._walk_error): + for name in filenames: + fpath = os.path.join(dirpath, name) + try: + total += os.path.getsize(fpath) + except OSError as exc: + self.warn("known-repos: could not check %s, reported cache size may be incomplete: %s", fpath, exc) + return total + + def _walk_error(self, exc): + self.warn("known-repos: could not list %s, reported cache size may be incomplete: %s", exc.filename, exc) + + +def _scan_security_root(scanner, root): + info = {} + for repo_id, entry in scanner.repo_id_dirs(root): + location_path = entry / "location" + key_type_path = entry / "key-type" + has_location = scanner.is_real_file(location_path) + has_key_type = scanner.is_real_file(key_type_path) + if not (has_location or has_key_type): + continue + location = scanner.read_text(location_path) if has_location else None + key_type_raw = scanner.read_text(key_type_path) if has_key_type else None + info[repo_id] = (location or None, (key_type_raw.strip() or None) if key_type_raw else None) + return info + + +def _scan_cache_root(scanner, root): + return {repo_id: entry for repo_id, entry in scanner.repo_id_dirs(root) if scanner.is_real_file(entry / "config")} + + +def _scan_keys_dir(scanner, root): + """Map repo_id -> set of key file paths, by reading only the first 128 bytes of each file + (enough to cover the "BORG_KEY " header, never the bulk of the encrypted key + material that follows it). A key file's name is a hash of its own content, not the repo + id, so this has to open every file in the directory - unlike the security/cache dirs, + which name their subdirectories after the repo id. + """ + info = {} + for _name, path in scanner.plain_files(root): + header = scanner.read_first_line(path) + if not header or not header.startswith(f"{KEYFILE_ID} "): + continue + repo_id = header[len(KEYFILE_ID) + 1 :].strip() + if not REPO_ID_RE.match(repo_id): + continue + info.setdefault(repo_id, set()).add(path) + return info + + +def _cache_layout(scanner, cache_entry): + """ "legacy" if the cache entry has a borg 1.x marker ("chunks.archive.d" or a "chunks" + file), else "borg2". A marker that could not be confirmed due to a filesystem error is + treated the same as absent (the error is already warned on by marker_exists()). + """ + if scanner.marker_exists(cache_entry / "chunks.archive.d"): + return "legacy" + return "legacy" if scanner.marker_exists(cache_entry / "chunks") else "borg2" + + +def _cache_previous_location(scanner, cache_entry): + config_text = scanner.read_text(cache_entry / "config") + if not config_text: + return None + parser = ConfigParser(interpolation=None) + try: + parser.read_string(config_text) + return parser.get("cache", "previous_location", fallback=None) or None + except ConfigParserError as exc: + scanner.warn("known-repos: could not parse %s: %s", cache_entry / "config", exc) + return None + + +class KnownReposMixIn: + def do_known_repos(self, args): + """List repositories Borg has local state for (security info and/or cache).""" + scanner = _Scanner() + + security_root = Path(get_security_dir(create=False)) + legacy_security_root = Path(get_security_dir_legacy(create=False)) + cache_root = Path(get_cache_dir(create=False)) + legacy_cache_root = Path(get_cache_dir_legacy(create=False)) + keys_root = Path(get_keys_dir(create=False)) + legacy_keys_root = Path(get_keys_dir_legacy(create=False)) + + security_roots_coincide = legacy_security_root == security_root + cache_roots_coincide = legacy_cache_root == cache_root + keys_roots_coincide = legacy_keys_root == keys_root + + borg2_security = _scan_security_root(scanner, security_root) + legacy_security = {} if security_roots_coincide else _scan_security_root(scanner, legacy_security_root) + + borg2_cache = _scan_cache_root(scanner, cache_root) + legacy_cache = {} if cache_roots_coincide else _scan_cache_root(scanner, legacy_cache_root) + legacy_cache = {k: v for k, v in legacy_cache.items() if k not in borg2_cache} + + keys = _scan_keys_dir(scanner, keys_root) + if not keys_roots_coincide: + for repo_id, paths in _scan_keys_dir(scanner, legacy_keys_root).items(): + keys.setdefault(repo_id, set()).update(paths) + + repo_ids = set(borg2_security) | set(legacy_security) | set(borg2_cache) | set(legacy_cache) | set(keys) + + repos = [] + for repo_id in sorted(repo_ids): + if repo_id in borg2_security: + location, key_type_raw = borg2_security[repo_id] + has_security_info = True + # location/key-type filenames are the same for borg1 and borg2, so a shared + # root cannot tell them apart - defer to the cache-based check below. + layout = None if security_roots_coincide else "borg2" + elif repo_id in legacy_security: + location, key_type_raw = legacy_security[repo_id] + layout = "legacy" + has_security_info = True + else: + location, key_type_raw = None, None + layout = None + has_security_info = False + + cache_entry = borg2_cache.get(repo_id) or legacy_cache.get(repo_id) + has_cache = cache_entry is not None + cache_size = None + cache_mtime = None + location_mismatch = False + if has_cache: + cache_size = scanner.dir_size(cache_entry) + cache_mtime = scanner.mtime(cache_entry / "config") + if layout is None: + if cache_roots_coincide: + layout = _cache_layout(scanner, cache_entry) + else: + layout = "legacy" if repo_id in legacy_cache else "borg2" + if layout == "legacy": + cache_location = _cache_previous_location(scanner, cache_entry) + if location is None: + location = cache_location + else: + # Compare the two stored location strings the same way + # assert_location_matches() does: normalize both sides' unicode form + # first (a no-op outside macOS), so on macOS an NFC vs. NFD difference + # alone isn't mistaken for a mismatch (#2913). + location_mismatch = cache_location is not None and normalize_local_path( + cache_location + ) != normalize_local_path(location) + + repos.append( + { + "id": repo_id, + "layout": layout, + "location": location, + "key_type": ( + (KEY_TYPE_LABELS.get(key_type_raw) or f"unknown key type {key_type_raw}") + if key_type_raw + else None + ), + "has_security_info": has_security_info, + "has_cache": has_cache, + "cache_size": cache_size, + "cache_config_mtime": cache_mtime.isoformat() if cache_mtime else None, + "key_file_count": len(keys.get(repo_id, ())), + "location_mismatch": location_mismatch, + } + ) + + if scanner.incomplete: + self.print_warning( + "known-repos: some filesystem inspections could not be completed; " + "results may be incomplete - see the warnings above." + ) + + if args.json: + json_print({"repositories": repos}) + return + + if not repos: + print("No locally known repositories.") + return + + headers = ("Repository ID", "Layout", "Cache", "Security", "Key files", "Last written") + rows = [] + for repo in repos: + security = "no" + if repo["has_security_info"]: + security = _printable(repo["key_type"]) if repo["key_type"] else "yes (key type unknown)" + location = _printable(repo["location"]) if repo["location"] else "(unknown)" + if repo["location_mismatch"]: + location += " (!= cache's last location)" + rows.append( + ( + repo["id"], + repo["layout"] or "unknown", + format_file_size(repo["cache_size"]) if repo["has_cache"] else "-", + security, + str(repo["key_file_count"]) if repo["key_file_count"] else "-", + repo["cache_config_mtime"][:19] if repo["cache_config_mtime"] else "-", + location, + ) + ) + + widths = [max(len(h), *(len(row[i]) for row in rows)) for i, h in enumerate(headers)] + print("".join(f"{h:<{w}} " for h, w in zip(headers, widths)) + "Location") + for row in rows: + print("".join(f"{col:<{w}} " for col, w in zip(row[:-1], widths)) + row[-1]) + + def build_parser_known_repos(self, subparsers, common_parser, mid_common_parser): + from ._common import process_epilog + + known_repos_epilog = process_epilog( + """ + Lists the repositories this local Borg installation currently has state for - + security info (key type, last known location), a local repository cache, and/or key + files - without accessing any repository itself (-r/--repo has no effect). "Layout" + is which state was found in (not the on-disk repository format, see "borg repo-info" + for that) - a security-only repo gets "unknown" when $BORG_SECURITY_DIR makes the + borg2 and legacy security roots coincide; "Last written" is the cache config's mtime, + a best-effort last-used proxy, not an exact timestamp and not a staleness check on its + own; a mismatch between a legacy repo's known location and its cache's last location + is flagged; a key file's presence does not by itself mean the repository is still + openable. Use --json for a machine-readable version. + """ + ) + subparser = ArgumentParser( + parents=[common_parser], description=self.do_known_repos.__doc__, epilog=known_repos_epilog + ) + subparsers.add_subcommand("known-repos", subparser, help="list locally known repositories") + subparser.add_argument("--json", action="store_true", help="format output as JSON") diff --git a/src/borg/legacy/fs.py b/src/borg/legacy/fs.py index 0939bfc908..3d3dca53cf 100644 --- a/src/borg/legacy/fs.py +++ b/src/borg/legacy/fs.py @@ -1,9 +1,11 @@ -"""borg 1.x directory layout, kept only so borg transfer can read v1 repos. +"""borg 1.x directory layout, kept only so borg transfer and known-repos can read v1 +repo state. The borg2 directory functions live in helpers.fs. This module keeps the old -home/base, config, cache, keys and security dir resolution. borg.security is -the only caller, and only when repository.version == 1. It goes away with the -rest of the borg 1.x transfer support. +home/base, config, cache, keys and security dir resolution. Callers are +borg.security (only when repository.version == 1) and +borg.archiver.known_repos_cmd (read-only, create=False). It goes away with +the rest of the borg 1.x transfer support. """ import os @@ -126,7 +128,7 @@ def get_keys_dir(*, create=True): keys_dir = os.environ.get("BORG_KEYS_DIR") if keys_dir is None: # note: do not just give this as default to the environment.get(), see issue #5979. - keys_dir = str(Path(get_config_dir()) / "keys") + keys_dir = str(Path(get_config_dir(create=create)) / "keys") if create: ensure_dir(keys_dir) return keys_dir @@ -137,7 +139,7 @@ def get_security_dir(repository_id=None, *, create=True): security_dir = os.environ.get("BORG_SECURITY_DIR") if security_dir is None: # note: do not just give this as default to the environment.get(), see issue #5979. - security_dir = str(Path(get_config_dir()) / "security") + security_dir = str(Path(get_config_dir(create=create)) / "security") if repository_id: security_dir = str(Path(security_dir) / repository_id) if create: diff --git a/src/borg/testsuite/archiver/known_repos_cmd_test.py b/src/borg/testsuite/archiver/known_repos_cmd_test.py new file mode 100644 index 0000000000..fad96e6d8e --- /dev/null +++ b/src/borg/testsuite/archiver/known_repos_cmd_test.py @@ -0,0 +1,464 @@ +import json +import os +import shutil +import unicodedata +from pathlib import Path + +import pytest + +from ...helpers import get_cache_dir, get_security_dir +from . import cmd, create_regular_file, exec_cmd, generate_archiver_tests, RK_ENCRYPTION, PURE_PYTHON_MSGPACK_WARNING + +pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA + + +def cmd_without_repo(archiver, *args): + """Like cmd(), but without the injected --repo option.""" + ret, output = exec_cmd(*args, archiver=archiver.archiver, fork=archiver.FORK_DEFAULT, exe=archiver.EXE) + assert ret == 0, output + # same as cmd(): drop the pure-python msgpack warning (e.g. on PyPy), it would break json.loads(). + return "".join(line for line in output.splitlines(keepends=True) if PURE_PYTHON_MSGPACK_WARNING not in line) + + +def _write_legacy_security(repo_id, location, key_type="3"): + """Create a borg-1.x-style $BORG_CONFIG_DIR/security// entry with the given + location and key-type (default "3" == KeyType.REPO, legacy AES-CTR repokey). + """ + from borg.legacy.fs import get_config_dir as get_config_dir_legacy + + security_dir = os.path.join(get_config_dir_legacy(create=False), "security", repo_id) + os.makedirs(security_dir, exist_ok=True) + with open(os.path.join(security_dir, "location"), "w") as fd: + fd.write(location) + with open(os.path.join(security_dir, "key-type"), "w") as fd: + fd.write(key_type) + + +def _write_legacy_cache(repo_id, previous_location=None): + """Create a legacy-layout ($BORG_CACHE_DIR//chunks marker) cache entry, optionally + with a [cache] previous_location in its config. + """ + cache_path = os.path.join(get_cache_dir(create=False), repo_id) + os.makedirs(cache_path, exist_ok=True) + with open(os.path.join(cache_path, "chunks"), "w") as fd: + fd.write("legacy chunks index placeholder") + if previous_location is not None: + with open(os.path.join(cache_path, "config"), "w") as fd: + fd.write(f"[cache]\nprevious_location = {previous_location}\n") + + +def test_known_repos_empty(archivers, request): + archiver = request.getfixturevalue(archivers) + result = cmd(archiver, "known-repos") + assert "No locally known repositories." in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + assert result_json == {"repositories": []} + + +def test_known_repos_without_repo_argument(archivers, request): + """The actual argv operators type - no -r/--repo at all - must work, not just cmd()'s injected one.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + output = cmd_without_repo(archiver, "known-repos", "--json") + ids = {repo["id"] for repo in json.loads(output)["repositories"]} + assert ids == {repo_id} + + +def test_known_repos_lists_repo_with_cache(archivers, request): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input") + + info = json.loads(cmd(archiver, "repo-info", "--json"))["repository"] + repo_id = info["id"] + + result = cmd(archiver, "known-repos") + assert repo_id in result.split() + assert "borg2" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos = result_json["repositories"] + assert len(repos) == 1 + repo = repos[0] + assert repo["id"] == repo_id + assert repo["layout"] == "borg2" + assert repo["has_cache"] is True + assert repo["cache_size"] > 0 + assert repo["key_type"] == "aes256-ocb, sha256" + assert repo["has_security_info"] is True + assert repo["cache_config_mtime"] is not None + assert repo["location"] == info["location"] + assert repo["key_file_count"] == 0 # repokey mode (the default) has no key file on disk + + # the table's "Last written" column shows the same cache_config_mtime, at/after its header + lines = result.splitlines() + header, row = lines[0], lines[1] + written_col = header.index("Last written") + assert repo["cache_config_mtime"][:10] in row[written_col:] # mtime's date part (YYYY-MM-DD) + + +def test_known_repos_cache_without_security_info(archivers, request): + """A cache dir with no matching security dir entry is still listed, with has_security_info False.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + security_path = get_security_dir(repo_id, create=False) + assert security_path.startswith(os.environ["BORG_BASE_DIR"] + os.sep) + shutil.rmtree(security_path, ignore_errors=True) + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["layout"] == "borg2" # discovered via the borg2 cache root even with no security info + assert repo["has_cache"] is True + assert repo["has_security_info"] is False + assert repo["key_type"] is None + assert repo["location"] is None + + +def test_known_repos_scans_legacy_v1_security_dir(archivers, request): + """A v1-style security entry (legacy $BORG_CONFIG_DIR/security layout) is listed too, with + no cache: the table's "Last written" column shows "-", not a blank or error. + """ + archiver = request.getfixturevalue(archivers) + legacy_repo_id = "a" * 64 # any syntactically valid 64-hex id; this test never opens a repo + _write_legacy_security(legacy_repo_id, "/some/v1/repo/path") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + legacy_repo = repos_by_id[legacy_repo_id] + assert legacy_repo["layout"] == "legacy" + assert legacy_repo["location"] == "/some/v1/repo/path" + assert legacy_repo["key_type"] == "aes256-ctr, sha256 (legacy)" + assert legacy_repo["has_cache"] is False + assert legacy_repo["cache_config_mtime"] is None + + table_result = cmd(archiver, "known-repos") + header, row = table_result.splitlines()[0], table_result.splitlines()[1] + written_col = header.index("Last written") + location_col = header.index("Location") + assert row[written_col:location_col].strip() == "-" + + +def test_known_repos_unknown_key_type_is_labeled(archivers, request): + """A key-type byte that does not match any known key class is shown as 'unknown key type N'.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + key_type_path = os.path.join(get_security_dir(repo_id, create=False), "key-type") + with open(key_type_path, "w") as fd: + fd.write("144") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["key_type"] == "unknown key type 144" + + +def test_known_repos_ignores_storecache_and_non_repo_dirs(archivers, request): + """A non-hex-id cache subdir (e.g. storecache), a stray security subdir, and a repo-id- + shaped name with a trailing newline (which must not match the repo-id pattern) are all + ignored. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + cache_root = get_cache_dir(create=False) + storecache_dir = f"{cache_root}/storecache" + os.makedirs(f"{storecache_dir}/subdir", exist_ok=True) + with open(f"{storecache_dir}/config", "w") as fd: + fd.write("not a repo cache config") + + security_root = get_security_dir(create=False) + stray_dir = f"{security_root}/not-a-repo-id" + os.makedirs(stray_dir, exist_ok=True) + with open(f"{stray_dir}/location", "w") as fd: + fd.write("should not be listed") + + if os.name != "nt": # Windows rejects a newline in a filename at creation time + bad_name = "a" * 64 + "\n" + bad_dir = os.path.join(security_root, bad_name) + os.makedirs(bad_dir, exist_ok=True) + with open(os.path.join(bad_dir, "location"), "w") as fd: + fd.write("/should/not/be/listed") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + ids = {repo["id"] for repo in result_json["repositories"]} + assert ids == {repo_id} + + +def test_known_repos_multiple_sorted_by_id(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id1 = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + second_repo = str(archiver.repository_path) + "-second" + cmd_without_repo(archiver, f"--repo={second_repo}", "repo-create", RK_ENCRYPTION) + info2 = json.loads(cmd_without_repo(archiver, f"--repo={second_repo}", "repo-info", "--json"))["repository"] + repo_id2 = info2["id"] + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + assert set(repos_by_id) == {repo_id1, repo_id2} + assert repos_by_id[repo_id2]["location"] == info2["location"] + assert [repo["id"] for repo in result_json["repositories"]] == sorted([repo_id1, repo_id2]) + + +def test_known_repos_does_not_write_cachedir_tag(archivers, request): + """Running known-repos on an otherwise-empty cache dir must not plant CACHEDIR.TAG there.""" + archiver = request.getfixturevalue(archivers) + cache_root = get_cache_dir(create=False) + cachedir_tag = os.path.join(cache_root, "CACHEDIR.TAG") + assert not os.path.exists(cachedir_tag) + + result = cmd(archiver, "known-repos") + assert "No locally known repositories." in result + assert not os.path.exists(cachedir_tag) + + +@pytest.mark.parametrize( + "setup", ["unreadable_security_entry", "security_root_not_a_directory", "unparseable_cache_config"] +) +def test_known_repos_warns_and_exits_nonzero_on_inspection_failure(archivers, request, monkeypatch, setup): + """Any filesystem inspection failure - an EACCES on a repo-id directory, a configured root + that is actually a plain file, or a cache config ConfigParser cannot parse - takes the + same path: it is skipped/treated as absent-looking, but the command still warns and exits + nonzero, rather than crashing or silently looking fully successful. + """ + archiver = request.getfixturevalue(archivers) + + if setup == "unreadable_security_entry": + if os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() == 0: + pytest.skip("chmod-based permission test, not meaningful on Windows or as root") + security_root = get_security_dir(create=False) + repo_id = "d" * 64 + security_path = os.path.join(security_root, repo_id) + os.makedirs(security_path, exist_ok=True) + with open(os.path.join(security_path, "location"), "w") as fd: + fd.write("/should/not/be/readable") + os.chmod(security_path, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be completed" in result + finally: + os.chmod(security_path, 0o755) + return + + if setup == "security_root_not_a_directory": + not_a_dir = str(Path(str(get_cache_dir(create=False))).parent / "not-a-directory") + with open(not_a_dir, "w") as fd: + fd.write("this is a file, not the security directory") + monkeypatch.setenv("BORG_SECURITY_DIR", not_a_dir) + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be completed" in result + return + + # setup == "unparseable_cache_config" + repo_id = "f" * 64 + cache_path = os.path.join(get_cache_dir(create=False), repo_id) + os.makedirs(cache_path, exist_ok=True) + with open(os.path.join(cache_path, "chunks"), "w") as fd: + fd.write("legacy chunks index placeholder") + with open(os.path.join(cache_path, "config"), "w") as fd: + fd.write("not a valid ini file [[[\n") + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be completed" in result + + +def test_known_repos_legacy_cache_previous_location_fallback(archivers, request): + """A legacy-only cache config's [cache] previous_location is used as location when there is + no security info for that id at all. The location contains a literal '%' (a valid path + character) to also verify it is returned verbatim, not raised on - ConfigParser's default + string interpolation treats '%' specially. + """ + archiver = request.getfixturevalue(archivers) + repo_id = "b" * 64 + _write_legacy_cache(repo_id, previous_location="/legacy/100%repo/path") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + repo = repos_by_id[repo_id] + assert repo["layout"] == "legacy" + assert repo["location"] == "/legacy/100%repo/path" + assert repo["has_security_info"] is False + + +def test_known_repos_discovers_legacy_cache_and_keys_in_separate_roots(archiver, monkeypatch, tmp_path): + """A legacy-only cache and key file are still discovered when their roots differ from + the borg2 ones (the macOS default). local-only: the binary archiver subprocess can't see + this monkeypatch. + """ + import borg.archiver.known_repos_cmd as known_repos_cmd + + separate_cache_root = tmp_path / "legacy-cache-root" + separate_keys_root = tmp_path / "legacy-keys-root" + monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create: str(separate_cache_root)) + monkeypatch.setattr(known_repos_cmd, "get_keys_dir_legacy", lambda create: str(separate_keys_root)) + + cache_repo_id = "b" * 64 + cache_path = separate_cache_root / cache_repo_id + cache_path.mkdir(parents=True) + (cache_path / "chunks").write_text("legacy chunks index placeholder") + (cache_path / "config").write_text("[cache]\nprevious_location = /legacy/v1/repo/path\n") + + key_repo_id = "f" * 64 + separate_keys_root.mkdir(parents=True) + (separate_keys_root / "keyfile.key").write_text(f"BORG_KEY {key_repo_id}\nrest of the key file\n") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + + cache_repo = repos_by_id[cache_repo_id] + assert cache_repo["layout"] == "legacy" + assert cache_repo["location"] == "/legacy/v1/repo/path" + assert cache_repo["has_security_info"] is False + + key_repo = repos_by_id[key_repo_id] + assert key_repo["key_file_count"] == 1 + assert key_repo["has_cache"] is False + + +@pytest.mark.parametrize( + "legacy_marker", + [ + "chunks", # legacy marker as a plain file + "chunks.archive.d", # legacy marker as a directory (checked first by _cache_layout()) + ], +) +def test_known_repos_layout_undetermined_when_security_roots_coincide(archivers, request, monkeypatch, legacy_marker): + """When $BORG_SECURITY_DIR makes the current and legacy security roots the same path, a + security entry alone cannot tell borg1 and borg2 apart (same "location"/"key-type" + filenames either way) - the cache content (chunks/chunks.archive.d) must decide the layout, + for either form the legacy marker can take. + """ + archiver = request.getfixturevalue(archivers) + security_root = get_security_dir(create=False) + monkeypatch.setenv("BORG_SECURITY_DIR", security_root) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + cache_root = get_cache_dir(create=False) + cache_path = os.path.join(cache_root, repo_id) + marker_path = os.path.join(cache_path, legacy_marker) + if legacy_marker.endswith(".d"): + os.makedirs(marker_path, exist_ok=True) + else: + with open(marker_path, "w") as fd: + fd.write("legacy chunks index placeholder") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["layout"] == "legacy" + + +def test_known_repos_table_columns_align_with_long_security_label(archivers, request): + """A Security label longer than the old fixed column width (e.g. an "authenticated-*" key + type with the "(legacy)" suffix) must not shift the Location column - the column widths + are computed from the actual data, not hardcoded. + """ + archiver = request.getfixturevalue(archivers) + security_root = get_security_dir(create=False) + repo_id = "c" * 64 + security_path = os.path.join(security_root, repo_id) + os.makedirs(security_path, exist_ok=True) + with open(os.path.join(security_path, "location"), "w") as fd: + fd.write("/some/repo") + with open(os.path.join(security_path, "key-type"), "w") as fd: + fd.write("7") # KeyType.AUTHENTICATED (legacy) -> "authenticated, sha256 (legacy)", 30 chars + + result = cmd(archiver, "known-repos") + lines = result.splitlines() + header, row = lines[0], lines[1] + location_col = header.index("Location") + assert row[location_col:].strip() == "/some/repo" + + +def test_known_repos_counts_keyfile_in_keys_dir(archivers, request): + """A key file in the keys directory is found by its plaintext header (not its filename, + which is a hash of the file's own content) and counted even after security info and the + cache for that repo are both gone - see #10453's "auditing ... keys" use case and + repo-delete, which deletes security info and cache but not the key file. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION, "--key-location=keyfile") + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["id"] == repo_id + assert repo["key_file_count"] == 1 + + cmd(archiver, "repo-delete", "--force") + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + repo = repos_by_id[repo_id] + assert repo["key_file_count"] == 1 + assert repo["has_security_info"] is False + assert repo["has_cache"] is False + + +def test_known_repos_ignores_non_key_files_in_keys_dir(archivers, request): + """A file in the keys directory without the BORG_KEY header is not mistaken for a key.""" + from borg.helpers import get_keys_dir + + archiver = request.getfixturevalue(archivers) + keys_dir = get_keys_dir(create=True) + create_regular_file(Path(keys_dir), "not_a_key_file", contents=b"just some unrelated file\n") + + result = cmd(archiver, "known-repos") + assert "No locally known repositories." in result + + +def test_known_repos_flags_location_mismatch_between_security_and_cache(archivers, request): + """A legacy repo whose security info has a location that differs from the cache config's + previous_location is flagged (location_mismatch) rather than silently preferring one over + the other - following the Bash workaround in #10453, which treated this as an error. A + cache previous_location that agrees with the security location is not flagged. + """ + archiver = request.getfixturevalue(archivers) + mismatched_id = "c" * 64 + _write_legacy_security(mismatched_id, "/correct/path") + _write_legacy_cache(mismatched_id, previous_location="/stale/different/path") + + agreeing_id = "d" * 64 + _write_legacy_security(agreeing_id, "/same/path") + _write_legacy_cache(agreeing_id, previous_location="/same/path") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + assert repos_by_id[mismatched_id]["location"] == "/correct/path" + assert repos_by_id[mismatched_id]["location_mismatch"] is True + assert repos_by_id[agreeing_id]["location_mismatch"] is False + + table_result = cmd(archiver, "known-repos") + assert "(!= cache's last location)" in table_result + + +def test_known_repos_location_mismatch_ignores_nfc_nfd_difference(archivers, request): + """Same path, different unicode normalization (NFC vs. NFD, see #2913) in the security + location and cache previous_location: not a mismatch on macOS (normalize_local_path() + normalizes both), but is one elsewhere (it's a no-op there). + """ + from borg.platformflags import is_darwin, is_win32 + + if is_win32: + pytest.skip("NFD path is not encodable in the Windows locale encoding") + + archiver = request.getfixturevalue(archivers) + nfc_path = unicodedata.normalize("NFC", "/absolute/tëst") + nfd_path = unicodedata.normalize("NFD", "/absolute/tëst") + assert nfc_path != nfd_path # sanity: the two forms really differ byte-wise + + repo_id = "e" * 64 + _write_legacy_security(repo_id, nfc_path) + _write_legacy_cache(repo_id, previous_location=nfd_path) + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = next(r for r in result_json["repositories"] if r["id"] == repo_id) + assert repo["location_mismatch"] is (not is_darwin) diff --git a/src/borg/testsuite/legacy_fs_test.py b/src/borg/testsuite/legacy_fs_test.py index 8dd41d5051..4eb08f38b9 100644 --- a/src/borg/testsuite/legacy_fs_test.py +++ b/src/borg/testsuite/legacy_fs_test.py @@ -12,6 +12,8 @@ get_base_dir as get_base_dir_legacy, get_config_dir as get_config_dir_legacy, get_cache_dir as get_cache_dir_legacy, + get_keys_dir as get_keys_dir_legacy, + get_security_dir as get_security_dir_legacy, ) from ..platform import is_win32, is_darwin, is_haiku, is_cygwin @@ -78,3 +80,32 @@ def test_get_cache_dir_compat(monkeypatch): assert get_cache_dir(create=False) == get_cache_dir_legacy(create=False) monkeypatch.setenv("BORG_CACHE_DIR", "/var/tmp/borg.cache.d") assert get_cache_dir(create=False) == get_cache_dir_legacy(create=False) + + +def test_get_keys_dir_does_not_create_config_dir(monkeypatch, tmp_path): + """get_keys_dir(create=False) must forward create=False to get_config_dir(), not create + the config dir as a side effect (regression). + """ + monkeypatch.delenv("BORG_KEYS_DIR", raising=False) + monkeypatch.delenv("BORG_CONFIG_DIR", raising=False) + monkeypatch.setenv("BORG_BASE_DIR", str(tmp_path)) + + config_dir = tmp_path / ".config" / "borg" + assert not config_dir.exists() + get_keys_dir_legacy(create=False) + assert not config_dir.exists() + + +def test_get_security_dir_does_not_create_config_dir(monkeypatch, tmp_path): + """Same regression as test_get_keys_dir_does_not_create_config_dir, for + get_security_dir(create=False) - what known_repos_cmd.py relies on for its read-only + promise. + """ + monkeypatch.delenv("BORG_SECURITY_DIR", raising=False) + monkeypatch.delenv("BORG_CONFIG_DIR", raising=False) + monkeypatch.setenv("BORG_BASE_DIR", str(tmp_path)) + + config_dir = tmp_path / ".config" / "borg" + assert not config_dir.exists() + get_security_dir_legacy(create=False) + assert not config_dir.exists()