From fe98739f71e14cf32aee8c99df2a60cf07724efc Mon Sep 17 00:00:00 2001 From: Kazuki Kanda <96773362+KazukiKandaKK@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:05:33 +0900 Subject: [PATCH 1/5] known-repos: new command to list locally known repositories Lists repositories this local Borg installation has state for (security info and/or a local cache) without accessing any repository, local or remote. Scans both the current borg2 security/cache directories and, separately, the legacy borg 1.x security/cache directories (which can be different default locations, e.g. on macOS). Does not follow symlinks anywhere in the scanned trees (repo-id directories, or the location/key-type files inside them), since borg only ever writes plain directories and regular files there. Key-type labels are derived from the key classes themselves (LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES), matching how "borg repo-info" names crypto suites, rather than a separately hand-maintained table. dir_size() pushes each directory handle (root, and any subdirectory opened while walking the cache tree) onto its explicit stack before calling list_names() on it, not after, so cleanup can always close it even if list_names() raises something other than OSError (it only catches OSError itself) - e.g. KeyboardInterrupt. Covered by two unit tests that monkeypatch list_names() to raise and then check the fd via os.fstat(). has_cache and has_security_info are tri-state (True/False/None, not a plain bool): an inspection failure (e.g. EACCES on a repo-id directory, or on the security/cache root itself, including a root that opens but fails during its own listing) is reported as unknown rather than being collapsed into a false "no". A repo-id directory that could be listed but not opened/classified is still included in the results with its fields set to unknown, instead of silently disappearing from the output. The JSON/text field for which state-directory layout a repository was discovered under is named "layout" (values "borg2" / "legacy" / null), not "version", so it cannot be confused with Borg's actual repository format version (see "borg repo-info"). Adds docs/usage/known-repos.rst{,.inc} and the usage.rst toctree entry, and the generated man page. --- docs/man/borg-known-repos.1 | 171 +++ docs/usage.rst | 1 + docs/usage/known-repos.rst | 30 + docs/usage/known-repos.rst.inc | 132 ++ src/borg/archiver/__init__.py | 3 + src/borg/archiver/known_repos_cmd.py | 756 ++++++++++++ .../archiver/known_repos_cmd_test.py | 1065 +++++++++++++++++ 7 files changed, 2158 insertions(+) create mode 100644 docs/man/borg-known-repos.1 create mode 100644 docs/usage/known-repos.rst create mode 100644 docs/usage/known-repos.rst.inc create mode 100644 src/borg/archiver/known_repos_cmd.py create mode 100644 src/borg/testsuite/archiver/known_repos_cmd_test.py diff --git a/docs/man/borg-known-repos.1 b/docs/man/borg-known-repos.1 new file mode 100644 index 0000000000..fde2eb8989 --- /dev/null +++ b/docs/man/borg-known-repos.1 @@ -0,0 +1,171 @@ +.\" Man page generated from reStructuredText +.\" by the Docutils 0.23 manpage writer. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "borg-known-repos" "1" "2026-10-10" "" "borg backup tool" +.SH Name +borg-known-repos \- List repositories Borg has local state for (security info and/or cache). +.SH SYNOPSIS +.sp +borg [common options] known\-repos [options] +.SH DESCRIPTION +.sp +This command lists the repositories that this local Borg installation currently +has state for: security information (key type, last known location) and/or a +local repository cache. +.sp +It inspects only local state \- the security directory and the cache directory, +each checked both at its current (borg2) default location and, separately, at +the legacy (borg 1.x) default location those same kinds of state used before the +borg2 base\-dir rewrite (see \fB$BORG_SECURITY_DIR\fP/\fB$BORG_CACHE_DIR\fP in +\fBborg help environment\fP for the exact resolution rules, including the platforms +where the two locations differ by default, e.g. macOS). It does not access any +repository, locally or remotely, so a repository that is currently unreachable +(e.g. a disconnected external disk or an unresponsive remote host) will still be +listed just fine. This also means \-r/\-\-repo has no effect on this command: it +always lists every locally known repository. +.sp +\(dqLayout\(dq reflects which state directory tree the repository was discovered in \- +\fBborg2\fP or \fBlegacy\fP \- not the repository\(aqs actual on\-disk format version (see +\fBborg repo\-info\fP\(aqs \(dqRepository version\(dq for that; this command never opens the +repository, so it cannot report that field at all). The borg2 security entry is +checked first, then the legacy (borg 1.x) security entry, then \- only when +neither security entry could be confirmed \- whichever cache root matched (borg2 +cache checked before the legacy cache). \(dqunknown\(dq means neither a security entry +nor a cache entry could be confirmed for this id \- it was listed as a directory +name, but this command could not establish which layout (if either) it belongs +to. If \fB$BORG_SECURITY_DIR\fP/\fB$BORG_CACHE_DIR\fP make the borg2 and legacy +locations resolve to the same path, only the borg2 scan runs and a legacy (v1) +repository there is reported with layout \fBborg2\fP\&. +.sp +This is useful for auditing which repositories this installation has local state for +(and, for repository\-encrypted modes, which crypto suite each one uses), finding +repositories with a local cache that may be taking up unwanted disk space, and general +housekeeping of stale local Borg state. +.sp +The reported cache size is the size of all regular files below the repository\(aqs +cache directory (symlinks are not followed and not counted) \- for a current (borg2) +repository that is the files cache plus a small amount of metadata (\fBconfig\fP, +\fBREADME\fP), not an index of the repository\(aqs chunks; a legacy (borg 1.x) cache +directory can be considerably larger, since it also holds a local chunks index. +A repository\-id directory that is a symlink (Borg itself always creates a plain +directory) is skipped, and a \fBlocation\fP or \fBkey\-type\fP file that is a symlink +is treated the same as missing; neither is followed. +.sp +\(dqCache last written\(dq is the modification time of the cache\(aqs own config file, as +a best\-effort proxy for when a cache\-using Borg command last ran against this +repository. It is not a precise \(dqlast backup\(dq or \(dqlast access\(dq time: not every +Borg command updates the cache, and a repository without a local cache at all +will not show one. +.sp +This command does not list or export any key material: the keys dir +(\fB$BORG_KEYS_DIR\fP) is not scanned. +.sp +\(dqSecurity info\(dq and \(dqLocal cache\(dq are each one of three states, not a plain +yes/no: \(dqyes\(dq (confirmed present \- for security info, a \fBlocation\fP and/or +\fBkey\-type\fP file was confirmed to exist there, independent of whether its +\fIcontent\fP could be read; for cache, a real, non\-symlinked cache directory with a +\fBconfig\fP file was confirmed to exist), \(dqno\(dq (confirmed absent \- this command +actually searched the relevant location(s) and found nothing there \- for the +cache specifically, the legacy root is only checked once the borg2 cache root is +confirmed absent for this id, so an unreadable borg2 cache entry reports +\(dqunknown\(dq rather than falling back to check whether a legacy cache exists), or +\(dqunknown\(dq +(this command could not determine which of the above is true \- most commonly a +permission error on the repository\-id directory itself, or on one of the files +inside it). A file that is simply missing (e.g. a repository with only a +\(dqlocation\(dq file and no \(dqkey\-type\(dq) does not affect the other field\(aqs state (that +repository still has \(dqSecurity info: yes\(dq); a security file that exists but +could not be fully read (a permission error, a file too large, or not valid +UTF\-8) does not change \(dqSecurity info\(dq away from \(dqyes\(dq either \- existence, not +readability, is what that field reports. A repository\-id directory that was found +during the directory listing but that then could not be opened or classified for +some other reason \- rather than being dropped from the results, as the directory +listing already proves it exists \- is still listed, with \(dqunknown\(dq for whichever +of \(dqSecurity info\(dq/\(dqLocal cache\(dq/\(dqLayout\(dq could not be established, so a script +auditing this output never mistakes \(dqcould not check\(dq for \(dqconfirmed not there\(dq. +For the cache specifically, once a cache directory\(aqs own existence is confirmed +(a real, non\-symlinked directory containing a \fBconfig\fP file), a later failure +to fully list its \fIcontents\fP does not change \(dqLocal cache\(dq away from \(dqyes\(dq \- +only the reported size becomes a partial count rather than the exact size. Any +of these inspection failures is both logged as its own warning and counted +towards a final summary warning, which (unlike a plain log message) makes this +command exit with a non\-zero return code (usually 1) \- including when the +failure happened while listing a security or cache root itself (e.g. EIO), which +can keep a repository that exists only there out of the output entirely, with no +per\-repository record to point to. A script checking the exit code therefore +learns only that the scan was incomplete, not that every affected repository is +still represented somewhere in the output. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS options +.INDENT 0.0 +.TP +.B \-\-json +format output as JSON +.UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +$ borg known\-repos +Repository ID: 0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 + Layout: borg2 + Location: /path/to/repo + Local cache: yes, 1.23 MB + Cache last written: 2026\-10\-07T21:00:00.000000+09:00 + Security info: yes (aes256\-ocb, sha256) + +$ borg known\-repos \-\-json +{ + \(dqrepositories\(dq: [ + { + \(dqcache_config_mtime\(dq: \(dq2026\-10\-07T21:00:00.000000+09:00\(dq, + \(dqcache_size\(dq: 1234567, + \(dqhas_cache\(dq: true, + \(dqhas_security_info\(dq: true, + \(dqid\(dq: \(dq0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903\(dq, + \(dqkey_type\(dq: \(dqaes256\-ocb, sha256\(dq, + \(dqkey_type_raw\(dq: \(dq16\(dq, + \(dqlayout\(dq: \(dqborg2\(dq, + \(dqlocation\(dq: \(dq/path/to/repo\(dq + } + ] +} +.EE +.UNINDENT +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH Author +The Borg Collective +.\" End of generated man page. diff --git a/docs/usage.rst b/docs/usage.rst index 02a74f95e2..ffc29721a4 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -44,6 +44,7 @@ Usage usage/repo-delete usage/serve usage/version + usage/known-repos usage/compact usage/lock usage/key diff --git a/docs/usage/known-repos.rst b/docs/usage/known-repos.rst new file mode 100644 index 0000000000..70320670a3 --- /dev/null +++ b/docs/usage/known-repos.rst @@ -0,0 +1,30 @@ +.. include:: known-repos.rst.inc + +Examples +~~~~~~~~ +:: + + $ borg known-repos + Repository ID: 0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 + Layout: borg2 + Location: /path/to/repo + Local cache: yes, 1.23 MB + Cache last written: 2026-10-07T21:00:00.000000+09:00 + Security info: yes (aes256-ocb, sha256) + + $ borg known-repos --json + { + "repositories": [ + { + "cache_config_mtime": "2026-10-07T21:00:00.000000+09:00", + "cache_size": 1234567, + "has_cache": true, + "has_security_info": true, + "id": "0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903", + "key_type": "aes256-ocb, sha256", + "key_type_raw": "16", + "layout": "borg2", + "location": "/path/to/repo" + } + ] + } diff --git a/docs/usage/known-repos.rst.inc b/docs/usage/known-repos.rst.inc new file mode 100644 index 0000000000..67dbf4eacc --- /dev/null +++ b/docs/usage/known-repos.rst.inc @@ -0,0 +1,132 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_known-repos: + +borg known-repos +---------------- +.. code-block:: none + + borg [common options] known-repos [options] + +.. only:: html + + .. class:: borg-options-table + + +-------------------------------------------------------+------------+-----------------------+ + | **options** | + +-------------------------------------------------------+------------+-----------------------+ + | | ``--json`` | format output as JSON | + +-------------------------------------------------------+------------+-----------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+------------+-----------------------+ + + .. raw:: html + + + +.. only:: latex + + + + options + --json format output as JSON + + + :ref:`common_options` + | + +Description +~~~~~~~~~~~ + +This command lists the repositories that this local Borg installation currently +has state for: security information (key type, last known location) and/or a +local repository cache. + +It inspects only local state - the security directory and the cache directory, +each checked both at its current (borg2) default location and, separately, at +the legacy (borg 1.x) default location those same kinds of state used before the +borg2 base-dir rewrite (see ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` in +``borg help environment`` for the exact resolution rules, including the platforms +where the two locations differ by default, e.g. macOS). It does not access any +repository, locally or remotely, so a repository that is currently unreachable +(e.g. a disconnected external disk or an unresponsive remote host) will still be +listed just fine. This also means -r/--repo has no effect on this command: it +always lists every locally known repository. + +"Layout" reflects which state directory tree the repository was discovered in - +``borg2`` or ``legacy`` - not the repository's actual on-disk format version (see +``borg repo-info``'s "Repository version" for that; this command never opens the +repository, so it cannot report that field at all). The borg2 security entry is +checked first, then the legacy (borg 1.x) security entry, then - only when +neither security entry could be confirmed - whichever cache root matched (borg2 +cache checked before the legacy cache). "unknown" means neither a security entry +nor a cache entry could be confirmed for this id - it was listed as a directory +name, but this command could not establish which layout (if either) it belongs +to. If ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` make the borg2 and legacy +locations resolve to the same path, only the borg2 scan runs and a legacy (v1) +repository there is reported with layout ``borg2``. + +This is useful for auditing which repositories this installation has local state for +(and, for repository-encrypted modes, which crypto suite each one uses), finding +repositories with a local cache that may be taking up unwanted disk space, and general +housekeeping of stale local Borg state. + +The reported cache size is the size of all regular files below the repository's +cache directory (symlinks are not followed and not counted) - for a current (borg2) +repository that is the files cache plus a small amount of metadata (``config``, +``README``), not an index of the repository's chunks; a legacy (borg 1.x) cache +directory can be considerably larger, since it also holds a local chunks index. +A repository-id directory that is a symlink (Borg itself always creates a plain +directory) is skipped, and a ``location`` or ``key-type`` file that is a symlink +is treated the same as missing; neither is followed. + +"Cache last written" is the modification time of the cache's own config file, as +a best-effort proxy for when a cache-using Borg command last ran against this +repository. It is not a precise "last backup" or "last access" time: not every +Borg command updates the cache, and a repository without a local cache at all +will not show one. + +This command does not list or export any key material: the keys dir +(``$BORG_KEYS_DIR``) is not scanned. + +"Security info" and "Local cache" are each one of three states, not a plain +yes/no: "yes" (confirmed present - for security info, a ``location`` and/or +``key-type`` file was confirmed to exist there, independent of whether its +*content* could be read; for cache, a real, non-symlinked cache directory with a +``config`` file was confirmed to exist), "no" (confirmed absent - this command +actually searched the relevant location(s) and found nothing there - for the +cache specifically, the legacy root is only checked once the borg2 cache root is +confirmed absent for this id, so an unreadable borg2 cache entry reports +"unknown" rather than falling back to check whether a legacy cache exists), or +"unknown" +(this command could not determine which of the above is true - most commonly a +permission error on the repository-id directory itself, or on one of the files +inside it). A file that is simply missing (e.g. a repository with only a +"location" file and no "key-type") does not affect the other field's state (that +repository still has "Security info: yes"); a security file that exists but +could not be fully read (a permission error, a file too large, or not valid +UTF-8) does not change "Security info" away from "yes" either - existence, not +readability, is what that field reports. A repository-id directory that was found +during the directory listing but that then could not be opened or classified for +some other reason - rather than being dropped from the results, as the directory +listing already proves it exists - is still listed, with "unknown" for whichever +of "Security info"/"Local cache"/"Layout" could not be established, so a script +auditing this output never mistakes "could not check" for "confirmed not there". +For the cache specifically, once a cache directory's own existence is confirmed +(a real, non-symlinked directory containing a ``config`` file), a later failure +to fully list its *contents* does not change "Local cache" away from "yes" - +only the reported size becomes a partial count rather than the exact size. Any +of these inspection failures is both logged as its own warning and counted +towards a final summary warning, which (unlike a plain log message) makes this +command exit with a non-zero return code (usually 1) - including when the +failure happened while listing a security or cache root itself (e.g. EIO), which +can keep a repository that exists only there out of the output entirely, with no +per-repository record to point to. A script checking the exit code therefore +learns only that the scan was incomplete, not that every affected repository is +still represented somewhere in the output. \ No newline at end of file diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index caa320611a..4bbd796da6 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -79,6 +79,7 @@ from .help_cmd import HelpMixIn from .info_cmd import InfoMixIn from .key_cmds import KeysMixIn +from .known_repos_cmd import KnownReposMixIn from .list_cmd import ListMixIn from .lock_cmds import LocksMixIn from .mount_cmds import MountMixIn @@ -116,6 +117,7 @@ class Archiver( HelpMixIn, InfoMixIn, KeysMixIn, + KnownReposMixIn, ListMixIn, LocksMixIn, MountMixIn, @@ -310,6 +312,7 @@ def build_parser(self): self.build_parser_help(subparsers, common_parser, mid_common_parser, parser) self.build_parser_info(subparsers, common_parser, mid_common_parser) self.build_parser_keys(subparsers, common_parser, mid_common_parser) + self.build_parser_known_repos(subparsers, common_parser, mid_common_parser) self.build_parser_list(subparsers, common_parser, mid_common_parser) self.build_parser_locks(subparsers, common_parser, mid_common_parser) self.build_parser_mount_umount(subparsers, common_parser, mid_common_parser) diff --git a/src/borg/archiver/known_repos_cmd.py b/src/borg/archiver/known_repos_cmd.py new file mode 100644 index 0000000000..9d948d4c02 --- /dev/null +++ b/src/borg/archiver/known_repos_cmd.py @@ -0,0 +1,756 @@ +import errno +import os +import re +import stat +from contextlib import ExitStack +from pathlib import Path + +from ..constants import * # NOQA +from ..crypto.key import LEGACY_KEY_TYPES, AVAILABLE_KEY_TYPES +from ..helpers import format_file_size, json_print, get_security_dir, get_cache_dir +from ..helpers import safe_timestamp, OutputTimestamp +from ..helpers import os_open, os_stat, flags_dir, flags_dir_follow, flags_normal +from ..helpers.argparsing import ArgumentParser +from ..legacy.fs import get_config_dir as get_config_dir_legacy +from ..legacy.fs import get_cache_dir as get_cache_dir_legacy + +from ..logger import create_logger + +logger = create_logger() + +# A borg2 repository id is a 32 byte random id, hex-encoded (bin_to_hex), see Repository.create(). +# \Z (not $) anchors strictly at the end of the string: $ also matches just before a trailing +# newline, which would let a directory name like "<64 hex chars>\n" slip through re.match(). +REPO_ID_RE = re.compile(r"^[0-9a-f]{64}\Z") + + +def _printable(s): + """Escape control characters for safe terminal output (JSON output is unaffected).""" + return "".join(c if c.isprintable() else repr(c)[1:-1] for c in s) + + +def _build_key_type_labels(): + """Map each key-type byte (as SecurityManager.save() writes it) to a readable label. + + Derived from LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES (the classes identify_key() searches), + matching how "borg repo-info" builds its "Encrypted:" line, rather than a hand-maintained table. + """ + labels = {} + for key_cls in LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES: + is_legacy = key_cls in LEGACY_KEY_TYPES + label = key_cls.ENC_NAME + if not key_cls.IDHASH_IN_ENC_NAME: + label += f", {key_cls.IDHASH_NAME}" + if is_legacy: + label += " (legacy)" + for type_byte in key_cls.TYPES_ACCEPTABLE: + labels[str(type_byte)] = label + return labels + + +# built once at import time: the key classes list is static. +KEY_TYPE_LABELS = _build_key_type_labels() + + +class _Inspector: + """Filesystem probes that count a failed probe as "incomplete" (see print_warning below). + + Built on borg's os_open()/os_stat() helpers (helpers/fs.py), the same ones create_cmd.py + uses. Where those return a real fd (not Windows), a validated directory is read from via + that fd rather than its pathname - closing the TOCTOU window where the pathname is replaced + between a check and a later read. On Windows, os_open() returns None for directories and a + pathname fallback is used instead - still correct, just not race-free. + """ + + def __init__(self): + self.incomplete = 0 + + def _warn(self, msg, *args): + self.incomplete += 1 + # escape control chars in str/OSError args before logging (logger.py has no sanitizer) + logger.warning(msg, *(_printable(str(a)) if isinstance(a, (str, OSError)) else a for a in args)) + + def close_dir(self, d): + """Close a directory handle from open_root/open_subdir - a no-op for the Windows + + pathname fallback (a Path, not a real fd). + """ + if not isinstance(d, Path): + os.close(d) + + def _open_dir(self, parent, name, display, follow_symlinks): + """Open name (an absolute Path if parent is None, else relative to the fd/Path parent) + + as a directory, via borg's os_open(). None if missing, not a directory, or (if not + follow_symlinks) a symlink. parent is None only for a scan root. The caller must + close_dir() a non-None result. + """ + flags = flags_dir_follow if follow_symlinks else flags_dir + try: + if parent is None: + fd = os_open(path=name, flags=flags) + elif isinstance(parent, Path): + fd = os_open(path=parent / name, flags=flags) # Windows pathname fallback + else: + fd = os_open(parent_fd=parent, name=name, flags=flags) + except (FileNotFoundError, NotADirectoryError): + return None + except OSError as exc: + if not follow_symlinks and getattr(exc, "errno", None) == errno.ELOOP: + return None # O_NOFOLLOW rejected a symlink leaf - not a failure, just "not a dir" + self._warn("known-repos: could not open %s: %s", display, exc) + return None + if fd is None: + # Windows: os_open() can't open a directory; fall back to the pathname itself. + path = name if parent is None else (parent / name if isinstance(parent, Path) else None) + if path is None: + self._warn("known-repos: could not open %s: no descriptor and no pathname fallback", display) + return None + try: + st = path.stat() if follow_symlinks else path.lstat() + except (FileNotFoundError, NotADirectoryError): + return None + except OSError as exc: + self._warn("known-repos: could not check %s: %s", display, exc) + return None + return path if stat.S_ISDIR(st.st_mode) else None + # release fd on every path out except the final success, including KeyboardInterrupt etc. + try: + is_real_dir = stat.S_ISDIR(os.fstat(fd).st_mode) + except OSError as exc: + os.close(fd) + self._warn("known-repos: could not check %s: %s", display, exc) + return None + except BaseException: + os.close(fd) + raise + if not is_real_dir: + os.close(fd) + return None + return fd + + def open_root(self, path, display): + """Open a scan root (security/cache root) by absolute path, following symlinks - these + + are Borg's own config/cache directories, not attacker-influenced repo-id entries, so + there is nothing to protect by rejecting a symlink here. None if not a real directory. + """ + return self._open_dir(None, path, display, follow_symlinks=True) + + def open_subdir(self, parent, name, display): + """Open and validate name below parent as a real (non-symlink) directory. None if + + missing/not-a-real-directory. The caller must close_dir() a non-None result. + """ + return self._open_dir(parent, name, display, follow_symlinks=False) + + def read_subfile(self, parent, name, display, max_size=4096): + """Read name below parent as a real (non-symlink) regular file, bounded. + + None if missing, not a regular file, too large, or not UTF-8. Missing and + "not a regular file" (including a symlink/FIFO in place of the expected file, + since Borg itself never creates one there) are not inspection failures; too + large and not-UTF-8 are, and so is a read/stat error on an entry that does + exist. + """ + if isinstance(parent, Path): + path = parent / name + try: + st = path.lstat() + except (FileNotFoundError, NotADirectoryError): + return None + except OSError as exc: + self._warn("known-repos: could not check %s: %s", display, exc) + return None + if not stat.S_ISREG(st.st_mode): + return None + try: + with path.open("rb") as f: + data = f.read(max_size + 1) + except (FileNotFoundError, NotADirectoryError): + return None + except OSError as exc: + self._warn("known-repos: could not read %s: %s", display, exc) + return None + else: + # flags_normal includes O_NONBLOCK|O_NOFOLLOW (see helpers/fs.py) - the same flags + # borg's own create_cmd.py uses to read a regular file without blocking on a FIFO + # that has no writer, and without following a symlink leaf. + try: + fd = os_open(parent_fd=parent, name=name, flags=flags_normal) + except (FileNotFoundError, NotADirectoryError): + return None + except OSError as exc: + if getattr(exc, "errno", None) == errno.ELOOP: + return None + self._warn("known-repos: could not read %s: %s", display, exc) + return None + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + return None + # loop until EOF or max_size + 1 bytes: a single os.read can return a short read + chunks = [] + remaining = max_size + 1 + while remaining > 0: + chunk = os.read(fd, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + except OSError as exc: + self._warn("known-repos: could not read %s: %s", display, exc) + return None + finally: + os.close(fd) + if len(data) > max_size: + self._warn("known-repos: %s is larger than %d bytes, ignoring its content", display, max_size) + return None + try: + return data.decode("utf-8") + except UnicodeDecodeError as exc: + self._warn("known-repos: %s is not valid UTF-8, ignoring its content: %s", display, exc) + return None + + def stat_subfile(self, parent, name, display): + """stat (not following symlinks) name below parent; the result only if it is a regular file.""" + if isinstance(parent, Path): + path = parent / name + try: + st = path.lstat() + except (FileNotFoundError, NotADirectoryError): + return None + except OSError as exc: + self._warn("known-repos: could not check %s: %s", display, exc) + return None + else: + try: + st = os_stat(parent_fd=parent, name=name, follow_symlinks=False) + except (FileNotFoundError, NotADirectoryError): + return None + except OSError as exc: + self._warn("known-repos: could not check %s: %s", display, exc) + return None + return st if stat.S_ISREG(st.st_mode) else None + + def list_names(self, d, display): + """List entry names in an open directory (fd or Path), or [] with a warning on OSError.""" + try: + if isinstance(d, Path): + return [e.name for e in d.iterdir()] + with os.scandir(d) as it: + return [e.name for e in it] + except (FileNotFoundError, NotADirectoryError): + return [] + except OSError as exc: + self._warn("known-repos: could not list %s, results may be incomplete: %s", display, exc) + return [] + + def dir_size(self, root, root_display): + """Sum regular-file sizes below an already-open/validated directory (fd or Path), not + + following symlinks. Consumes (closes) root. Best-effort: an unreadable entry is + skipped (warned, counted incomplete), so the total can be a partial count. Uses an + explicit stack, not Python recursion, to bound open descriptors by tree depth, not + breadth, and to avoid Python's recursion limit on a very deep tree. + """ + total = 0 + # each entry is (dir_handle, display, remaining_names_iterator); closed once exhausted. + # root is pushed before the try so a BaseException from list_names() (list_names() itself + # catches OSError) still leaves root on the stack for the finally below to close. + stack = [[root, root_display, None]] + try: + stack[0][2] = iter(self.list_names(root, root_display)) + while stack: + current, display, names = stack[-1] + name = next(names, None) + if name is None: + self.close_dir(current) + stack.pop() + continue + entry_display = f"{display}/{name}" + if isinstance(current, Path): + try: + st = (current / name).lstat() + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as exc: + self._warn( + "known-repos: could not check %s, reported cache size may be incomplete: %s", + entry_display, + exc, + ) + continue + else: + try: + st = os_stat(parent_fd=current, name=name, follow_symlinks=False) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as exc: + self._warn( + "known-repos: could not check %s, reported cache size may be incomplete: %s", + entry_display, + exc, + ) + continue + if stat.S_ISDIR(st.st_mode): + sub = self.open_subdir(current, name, entry_display) + if sub is not None: + frame = [sub, entry_display, None] + stack.append(frame) + frame[2] = iter(self.list_names(sub, entry_display)) + elif stat.S_ISREG(st.st_mode): + total += st.st_size + finally: + # closes anything left on the stack on an exception path (ordinary completion + # already pops/closes each level). + for entry in stack: + self.close_dir(entry[0]) + return total + + def cache_config_mtime(self, cache_dir, display): + """mtime of config below an already-open cache_dir, as an OutputTimestamp, or None. + + Best-effort proxy for "a cache-using borg command last ran" - not every command touches + the cache, so this can lag behind actual repository access. + """ + st = self.stat_subfile(cache_dir, "config", f"{display}/config") + if st is None: + return None + try: + return OutputTimestamp(safe_timestamp(st.st_mtime_ns)) + except (OverflowError, OSError, ValueError) as exc: + self._warn("known-repos: could not convert the mtime of %s/config: %s", display, exc) + return None + + def open_cache_entry(self, cache_root, repo_id, display): + """Open and classify the cache entry for repo_id below an already-open cache_root. + + Returns (kind, entry): 'present' (entry is the open, caller-owned handle for + dir_size/cache_config_mtime), 'absent' (confirmed not a cache, entry is None), or + 'unreadable' (state never established - not a confirmed cache, and the caller must not + fall back to another root either; entry is None). + """ + before = self.incomplete + entry = self.open_subdir(cache_root, repo_id, display) + if entry is None: + return ("unreadable" if self.incomplete > before else "absent"), None + # release entry on every path out except the "present" return, which transfers ownership + try: + before = self.incomplete + if self.stat_subfile(entry, "config", f"{display}/config") is not None: + return "present", entry + kind = "unreadable" if self.incomplete > before else "absent" + except BaseException: + self.close_dir(entry) + raise + self.close_dir(entry) + return kind, None + + def scan_security_root(self, root, root_display): + """Return (repo_ids, info, unreadable_ids, listing_ok) for ids directly below an + already-open security root. + + repo_ids is the set of 64-hex-digit directories with at least one of the two known files + confirmed present (checked via stat_subfile, independent of read success). info is + {repo_id: (location, key_type_raw)}, read while the entry is open; a value is None if + its file is absent or could not be read/decoded. + + unreadable_ids is disjoint from repo_ids: directory names list_names() returned whose + presence/absence could not be established (the entry itself could not be opened, e.g. + EACCES, or neither file was confirmed present and at least one of the two + presence checks failed). A name list_names() returned must end up in repo_ids, + unreadable_ids, or genuinely neither - never silently dropped. + + listing_ok is False when list_names() on root itself failed (e.g. EIO after the root was + already opened): repo_ids/unreadable_ids are then necessarily incomplete, so the caller + must not treat this root as fully searched for an id discovered elsewhere. + """ + repo_ids = set() + info = {} + unreadable_ids = set() + before_list = self.incomplete + names = self.list_names(root, root_display) + listing_ok = self.incomplete == before_list + for name in names: + if not REPO_ID_RE.match(name): + continue + entry_display = f"{root_display}/{name}" + before = self.incomplete + entry = self.open_subdir(root, name, entry_display) + if entry is None: + if self.incomplete > before: + unreadable_ids.add(name) + continue + try: + before = self.incomplete + has_location = self.stat_subfile(entry, "location", f"{entry_display}/location") is not None + has_key_type = self.stat_subfile(entry, "key-type", f"{entry_display}/key-type") is not None + if has_location or has_key_type: + location = self.read_subfile(entry, "location", f"{entry_display}/location") + # SecurityManager.save() writes location verbatim (no trailing newline); only + # turn a present-but-empty read into None, do not alter legitimate whitespace. + location = location or None + key_type_raw = self.read_subfile(entry, "key-type", f"{entry_display}/key-type") + key_type_raw = key_type_raw.strip() or None if key_type_raw else None + repo_ids.add(name) + info[name] = (location, key_type_raw) + elif self.incomplete > before: + # stat_subfile only counts incomplete on a real OSError, not on + # FileNotFoundError/NotADirectoryError, so this path means a genuine + # EACCES-class failure, not plain absence. + unreadable_ids.add(name) + # else: neither file exists - not a security entry, not an inspection failure. + finally: + self.close_dir(entry) + return repo_ids, info, unreadable_ids, listing_ok + + def scan_cache_root(self, root, root_display, repo_ids_out, unreadable_ids_out): + """Add repo ids directly below an already-open cache root into repo_ids_out: a + + 64-hex-digit directory with a config *file* (excludes the storecache directory and + the CACHEDIR.TAG file get_cache_dir() writes). A name list_names() returned that could + not be classified as having a cache or not (the entry could not be opened, or its + config file's presence could not be established) goes into unreadable_ids_out instead + of being dropped - see scan_security_root() for why that distinction matters. + + Returns listing_ok (see scan_security_root() for its meaning). + """ + before_list = self.incomplete + names = self.list_names(root, root_display) + listing_ok = self.incomplete == before_list + for name in names: + if not REPO_ID_RE.match(name): + continue + entry_display = f"{root_display}/{name}" + before = self.incomplete + entry = self.open_subdir(root, name, entry_display) + if entry is None: + if self.incomplete > before: + unreadable_ids_out.add(name) + continue + try: + before = self.incomplete + is_cache = self.stat_subfile(entry, "config", f"{entry_display}/config") is not None + finally: + self.close_dir(entry) + if is_cache: + repo_ids_out.add(name) + elif self.incomplete > before: + unreadable_ids_out.add(name) + return listing_ok + + +def _legacy_security_dir(): + """Where borg 1.x stores local security information, without creating anything. + + Mirrors legacy.fs.get_security_dir(create=False), but that still calls get_config_dir() + with create=True internally; re-implemented here to stay fully read-only. + """ + security_dir = os.environ.get("BORG_SECURITY_DIR") + if security_dir is None: + security_dir = str(Path(get_config_dir_legacy(create=False)) / "security") + return security_dir + + +class KnownReposMixIn: + def do_known_repos(self, args): + """List repositories Borg has local state for (security info and/or cache).""" + inspector = _Inspector() + security_root = Path(get_security_dir(create=False)) + # v1 repos keep security info at a different default location than borg2 (and on some + # platforms, e.g. macOS, a different cache root too) unless BORG_SECURITY_DIR/BORG_CACHE_DIR + # override both to the same place. + legacy_security_root = Path(_legacy_security_dir()) + cache_root = Path(get_cache_dir(create=False)) + legacy_cache_root = Path(get_cache_dir_legacy(create=False)) + + with ExitStack() as stack: + # every root is opened once up front and held open for the whole scan. + def _open(path): + before = inspector.incomplete + d = inspector.open_root(path, str(path)) + if d is not None: + stack.callback(inspector.close_dir, d) + return d, "present" + return None, ("unreadable" if inspector.incomplete > before else "absent") + + security_root_d, security_root_kind = _open(security_root) + if legacy_security_root != security_root: + legacy_security_root_d, legacy_security_root_kind = _open(legacy_security_root) + else: + legacy_security_root_d, legacy_security_root_kind = None, "absent" + cache_root_d, cache_root_kind = _open(cache_root) + if legacy_cache_root != cache_root: + legacy_cache_root_d, legacy_cache_root_kind = _open(legacy_cache_root) + else: + legacy_cache_root_d, legacy_cache_root_kind = None, "absent" + + if security_root_d is not None: + borg2_ids, borg2_info, borg2_unreadable_ids, borg2_security_listing_ok = inspector.scan_security_root( + security_root_d, str(security_root) + ) + else: + borg2_ids, borg2_info, borg2_unreadable_ids, borg2_security_listing_ok = set(), {}, set(), True + if legacy_security_root_d is not None: + legacy_ids, legacy_info, legacy_unreadable_ids, legacy_security_listing_ok = ( + inspector.scan_security_root(legacy_security_root_d, str(legacy_security_root)) + ) + legacy_ids -= borg2_ids # prefer the borg2 entry if an id somehow exists in both trees + legacy_unreadable_ids -= borg2_ids + else: + legacy_ids, legacy_info, legacy_unreadable_ids, legacy_security_listing_ok = set(), {}, set(), True + security_unreadable_ids = (borg2_unreadable_ids | legacy_unreadable_ids) - borg2_ids - legacy_ids + repo_ids = borg2_ids | legacy_ids | security_unreadable_ids + # "no security entry" can only be a confirmed False for a cache-only id when BOTH + # security roots were fully searched (root opened and its own listing succeeded, + # not just free of per-id EACCES - scan_security_root() tracks that separately). + security_fully_searched = ( + security_root_kind != "unreadable" + and legacy_security_root_kind != "unreadable" + and borg2_security_listing_ok + and legacy_security_listing_ok + ) + borg2_cache_ids = set() + borg2_cache_unreadable_ids = set() + if cache_root_d is not None: + inspector.scan_cache_root(cache_root_d, str(cache_root), borg2_cache_ids, borg2_cache_unreadable_ids) + legacy_cache_ids = set() + legacy_cache_unreadable_ids = set() + if legacy_cache_root_d is not None: + inspector.scan_cache_root( + legacy_cache_root_d, str(legacy_cache_root), legacy_cache_ids, legacy_cache_unreadable_ids + ) + cache_unreadable_ids = (borg2_cache_unreadable_ids | legacy_cache_unreadable_ids) - ( + borg2_cache_ids | legacy_cache_ids + ) + repo_ids |= borg2_cache_ids | legacy_cache_ids | cache_unreadable_ids + + repos = [] + for repo_id in sorted(repo_ids): + if repo_id in borg2_ids: + location, key_type_raw = borg2_info[repo_id] + layout = "borg2" + has_security_info = True + elif repo_id in legacy_ids: + location, key_type_raw = legacy_info[repo_id] + layout = "legacy" + has_security_info = True + elif repo_id in borg2_cache_ids: + location, key_type_raw = None, None + layout = "borg2" + # discovered only via the cache; security info is confirmed absent only if + # this id was not flagged unreadable AND both security roots were fully + # searched (see security_fully_searched above). + if repo_id in security_unreadable_ids or not security_fully_searched: + has_security_info = None + else: + has_security_info = False + elif repo_id in legacy_cache_ids: + location, key_type_raw = None, None + layout = "legacy" + if repo_id in security_unreadable_ids or not security_fully_searched: + has_security_info = None + else: + has_security_info = False + else: + location, key_type_raw = None, None + layout = None + if repo_id in security_unreadable_ids or not security_fully_searched: + has_security_info = None + else: + has_security_info = False + + # fall back to the legacy cache root only when borg2 is confirmed absent - + # "unreadable" must not be treated as "try the other root" nor as "present". + used_root_path = cache_root + if cache_root_d is not None: + kind, entry = inspector.open_cache_entry(cache_root_d, repo_id, str(cache_root / repo_id)) + else: + kind, entry = cache_root_kind, None + if kind == "absent" and legacy_cache_root_d is not None: + used_root_path = legacy_cache_root + kind, entry = inspector.open_cache_entry( + legacy_cache_root_d, repo_id, str(legacy_cache_root / repo_id) + ) + elif kind == "absent" and legacy_cache_root_kind == "unreadable": + # current-root confirmed absent, but the legacy root could not be opened + # either - the legacy side was never searched, so this is unknown, not absent. + kind = "unreadable" + + # tri-state: True (confirmed present), False (confirmed absent in every root + # that could be searched), or None (at least one root's state could not be + # established). This re-examines repo_id's cache state fresh rather than + # reusing scan_cache_root()'s earlier discovery pass. + has_cache = {"present": True, "unreadable": None, "absent": False}[kind] + cache_size = None + cache_mtime = None + if has_cache is True: + # dir_size closes entry unconditionally; guard only the window before it + # starts, where cache_config_mtime could raise first. + entry_display = str(used_root_path / repo_id) + try: + cache_mtime = inspector.cache_config_mtime(entry, entry_display) + except BaseException: + inspector.close_dir(entry) + raise + cache_size = inspector.dir_size(entry, entry_display) # consumes/closes entry; always an int + elif entry is not None: + inspector.close_dir(entry) + + repos.append( + { + "id": repo_id, + "layout": layout, + "location": location, + "key_type_raw": key_type_raw, + "key_type": (KEY_TYPE_LABELS.get(key_type_raw) or key_type_raw) if key_type_raw else None, + "has_security_info": has_security_info, + "has_cache": has_cache, + "cache_size": cache_size if has_cache is True else None, + "cache_config_mtime": cache_mtime.isoformat() if has_cache is True and cache_mtime else None, + } + ) + + if inspector.incomplete: + # counts failed inspection operations, not distinct paths (the same path can be + # probed more than once), so this number can exceed the paths actually affected. + self.print_warning( + "known-repos: %d filesystem inspection(s) could not be fully completed; " + "results may be incomplete - see the warnings above.", + inspector.incomplete, + ) + + if args.json: + json_print({"repositories": repos}) + return + + if not repos: + print("No locally known repositories.") + return + + for repo in repos: + print(f"Repository ID: {repo['id']}") + layout = repo["layout"] + if layout is None: + print(" Layout: unknown (could not be determined)") + else: + print(f" Layout: {layout}") + print(f" Location: {_printable(repo['location']) if repo['location'] else '(unknown)'}") + if repo["has_cache"] is True: + print(f" Local cache: yes, {format_file_size(repo['cache_size'])}") + print(f" Cache last written: {repo['cache_config_mtime'] or '(unknown)'}") + elif repo["has_cache"] is None: + print(" Local cache: unknown (could not be determined)") + else: + print(" Local cache: no") + if repo["has_security_info"] is True: + key_type = repo["key_type"] + if key_type: + print(f" Security info: yes ({_printable(key_type)})") + else: + print(" Security info: yes (key type unknown)") + elif repo["has_security_info"] is None: + print(" Security info: unknown (could not be determined)") + else: + print(" Security info: no") + if repo["has_cache"] is True and repo["has_security_info"] is False: + print(" Note: a local cache exists, but no security info was found for this repository.") + print() + + def build_parser_known_repos(self, subparsers, common_parser, mid_common_parser): + from ._common import process_epilog + + known_repos_epilog = process_epilog(""" + This command lists the repositories that this local Borg installation currently + has state for: security information (key type, last known location) and/or a + local repository cache. + + It inspects only local state - the security directory and the cache directory, + each checked both at its current (borg2) default location and, separately, at + the legacy (borg 1.x) default location those same kinds of state used before the + borg2 base-dir rewrite (see ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` in + ``borg help environment`` for the exact resolution rules, including the platforms + where the two locations differ by default, e.g. macOS). It does not access any + repository, locally or remotely, so a repository that is currently unreachable + (e.g. a disconnected external disk or an unresponsive remote host) will still be + listed just fine. This also means -r/--repo has no effect on this command: it + always lists every locally known repository. + + "Layout" reflects which state directory tree the repository was discovered in - + ``borg2`` or ``legacy`` - not the repository's actual on-disk format version (see + ``borg repo-info``'s "Repository version" for that; this command never opens the + repository, so it cannot report that field at all). The borg2 security entry is + checked first, then the legacy (borg 1.x) security entry, then - only when + neither security entry could be confirmed - whichever cache root matched (borg2 + cache checked before the legacy cache). "unknown" means neither a security entry + nor a cache entry could be confirmed for this id - it was listed as a directory + name, but this command could not establish which layout (if either) it belongs + to. If ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` make the borg2 and legacy + locations resolve to the same path, only the borg2 scan runs and a legacy (v1) + repository there is reported with layout ``borg2``. + + This is useful for auditing which repositories this installation has local state for + (and, for repository-encrypted modes, which crypto suite each one uses), finding + repositories with a local cache that may be taking up unwanted disk space, and general + housekeeping of stale local Borg state. + + The reported cache size is the size of all regular files below the repository's + cache directory (symlinks are not followed and not counted) - for a current (borg2) + repository that is the files cache plus a small amount of metadata (``config``, + ``README``), not an index of the repository's chunks; a legacy (borg 1.x) cache + directory can be considerably larger, since it also holds a local chunks index. + A repository-id directory that is a symlink (Borg itself always creates a plain + directory) is skipped, and a ``location`` or ``key-type`` file that is a symlink + is treated the same as missing; neither is followed. + + "Cache last written" is the modification time of the cache's own config file, as + a best-effort proxy for when a cache-using Borg command last ran against this + repository. It is not a precise "last backup" or "last access" time: not every + Borg command updates the cache, and a repository without a local cache at all + will not show one. + + This command does not list or export any key material: the keys dir + (``$BORG_KEYS_DIR``) is not scanned. + + "Security info" and "Local cache" are each one of three states, not a plain + yes/no: "yes" (confirmed present - for security info, a ``location`` and/or + ``key-type`` file was confirmed to exist there, independent of whether its + *content* could be read; for cache, a real, non-symlinked cache directory with a + ``config`` file was confirmed to exist), "no" (confirmed absent - this command + actually searched the relevant location(s) and found nothing there - for the + cache specifically, the legacy root is only checked once the borg2 cache root is + confirmed absent for this id, so an unreadable borg2 cache entry reports + "unknown" rather than falling back to check whether a legacy cache exists), or + "unknown" + (this command could not determine which of the above is true - most commonly a + permission error on the repository-id directory itself, or on one of the files + inside it). A file that is simply missing (e.g. a repository with only a + "location" file and no "key-type") does not affect the other field's state (that + repository still has "Security info: yes"); a security file that exists but + could not be fully read (a permission error, a file too large, or not valid + UTF-8) does not change "Security info" away from "yes" either - existence, not + readability, is what that field reports. A repository-id directory that was found + during the directory listing but that then could not be opened or classified for + some other reason - rather than being dropped from the results, as the directory + listing already proves it exists - is still listed, with "unknown" for whichever + of "Security info"/"Local cache"/"Layout" could not be established, so a script + auditing this output never mistakes "could not check" for "confirmed not there". + For the cache specifically, once a cache directory's own existence is confirmed + (a real, non-symlinked directory containing a ``config`` file), a later failure + to fully list its *contents* does not change "Local cache" away from "yes" - + only the reported size becomes a partial count rather than the exact size. Any + of these inspection failures is both logged as its own warning and counted + towards a final summary warning, which (unlike a plain log message) makes this + command exit with a non-zero return code (usually 1) - including when the + failure happened while listing a security or cache root itself (e.g. EIO), which + can keep a repository that exists only there out of the output entirely, with no + per-repository record to point to. A script checking the exit code therefore + learns only that the scan was incomplete, not that every affected repository is + still represented somewhere in the output. + """) + subparser = ArgumentParser( + parents=[common_parser], description=self.do_known_repos.__doc__, epilog=known_repos_epilog + ) + subparsers.add_subcommand("known-repos", subparser, help="list locally known repositories") + subparser.add_argument("--json", action="store_true", help="format output as JSON") diff --git a/src/borg/testsuite/archiver/known_repos_cmd_test.py b/src/borg/testsuite/archiver/known_repos_cmd_test.py new file mode 100644 index 0000000000..77f15e754f --- /dev/null +++ b/src/borg/testsuite/archiver/known_repos_cmd_test.py @@ -0,0 +1,1065 @@ +import json +import os +import shutil + +try: + import resource +except ImportError: # Windows has no resource module + resource = None + +import pytest + +from ...constants import * # NOQA +from ...helpers import get_cache_dir, get_security_dir, format_file_size +from .. import are_symlinks_supported +from . import cmd, create_regular_file, exec_cmd, generate_archiver_tests, RK_ENCRYPTION + +pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA + +requires_symlinks = pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported") +# evaluate geteuid only on POSIX: the name does not exist on Windows, and putting +# os.geteuid() in a skipif condition would raise at collection time there. +requires_chmod_permissions = pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() == 0, + reason="chmod-based permission test, not meaningful on Windows or as root", +) + + +def test_known_repos_empty(archivers, request): + archiver = request.getfixturevalue(archivers) + result = cmd(archiver, "known-repos") + assert "No locally known repositories." in result + + +def test_known_repos_without_repo_argument(archivers, request): + """The actual argv operators type - no -r/--repo at all - must work, not just cmd()'s injected one.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + ret, output = exec_cmd( + "known-repos", "--json", archiver=archiver.archiver, fork=archiver.FORK_DEFAULT, exe=archiver.EXE + ) + assert ret == 0, output + ids = {repo["id"] for repo in json.loads(output)["repositories"]} + assert ids == {repo_id} + + +def test_known_repos_empty_json(archivers, request): + archiver = request.getfixturevalue(archivers) + result = json.loads(cmd(archiver, "known-repos", "--json")) + assert result == {"repositories": []} + + +def test_known_repos_lists_repo_with_cache(archivers, request): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input") + + info = json.loads(cmd(archiver, "repo-info", "--json"))["repository"] + repo_id = info["id"] + + result = cmd(archiver, "known-repos") + assert f"Repository ID: {repo_id}" in result + assert "Layout: borg2" in result + assert "Local cache: yes" in result + assert "Cache last written:" in result + assert "Security info: yes (aes256-ocb, sha256)" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos = result_json["repositories"] + assert len(repos) == 1 + repo = repos[0] + assert repo["id"] == repo_id + assert repo["layout"] == "borg2" + assert repo["has_cache"] is True + assert repo["cache_size"] > 0 + assert repo["key_type"] == "aes256-ocb, sha256" + assert repo["key_type_raw"] is not None + assert repo["has_security_info"] is True + assert repo["cache_config_mtime"] is not None + assert repo["location"] == info["location"] + + +def test_known_repos_without_cache(archivers, request): + """A repo whose local cache was deleted (but security info remains) is still reported, without a cache.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + cache_path = get_cache_dir(repo_id, create=False) + # sanity check: only remove a cache dir that is really below this test's isolated BORG_CACHE_DIR + # (the archiver fixture points BORG_CACHE_DIR at archiver.cache_path, a per-test tmp dir), so a + # misconfigured test environment can not delete a real, unrelated local cache. + assert cache_path.startswith(str(archiver.cache_path) + os.sep) + shutil.rmtree(cache_path, ignore_errors=True) + + result = cmd(archiver, "known-repos") + assert "Local cache: no" in result + assert "Note: a local cache exists" not in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos = result_json["repositories"] + assert len(repos) == 1 + repo = repos[0] + assert repo["id"] == repo_id + assert repo["has_cache"] is False + assert repo["cache_size"] is None + assert repo["cache_config_mtime"] is None + # security info (key type, location) must still be reported even without a cache + assert repo["key_type"] == "aes256-ocb, sha256" + assert repo["has_security_info"] is True + assert repo["location"] + + +def test_known_repos_cache_without_security_info(archivers, request): + """A cache dir with no matching security dir entry is still listed, with the "no security info" note.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + security_path = get_security_dir(repo_id, create=False) + assert security_path.startswith(os.environ["BORG_BASE_DIR"] + os.sep) # same isolation sanity check as above + shutil.rmtree(security_path, ignore_errors=True) + + result = cmd(archiver, "known-repos") + assert f"Repository ID: {repo_id}" in result + assert "Layout: borg2" in result # discovered via the borg2 cache root even with no security info + assert "Local cache: yes" in result + assert "Security info: no" in result + assert "Note: a local cache exists, but no security info was found" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["layout"] == "borg2" + assert repo["has_cache"] is True + assert repo["has_security_info"] is False + assert repo["key_type"] is None + assert repo["location"] is None + + +def test_known_repos_location_only_counts_as_security_info(archivers, request): + """A security dir with only a 'location' file (no readable key-type) still counts as having info.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + security_path = get_security_dir(repo_id, create=False) + assert security_path.startswith(os.environ["BORG_BASE_DIR"] + os.sep) + os.remove(os.path.join(security_path, "key-type")) + + result = cmd(archiver, "known-repos") + assert "Security info: yes (key type unknown)" in result + assert "Note: a local cache exists" not in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["has_security_info"] is True + assert repo["key_type"] is None + assert repo["key_type_raw"] is None + assert repo["location"] + + +def test_known_repos_ignores_storecache_and_non_repo_dirs(archivers, request): + """A non-hex-id cache subdir (e.g. storecache) and a stray security subdir are not listed.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + cache_root = get_cache_dir(create=False) + # BORG_STORE_CACHE's storecache dir has no "config" file of its own, but even one placed there + # deliberately must not make it count, since it is not a 64-hex-digit directory name. + storecache_dir = f"{cache_root}/storecache" + os.makedirs(f"{storecache_dir}/subdir", exist_ok=True) + with open(f"{storecache_dir}/config", "w") as fd: + fd.write("not a repo cache config") + + security_root = get_security_dir(create=False) + stray_dir = f"{security_root}/not-a-repo-id" + os.makedirs(stray_dir, exist_ok=True) + with open(f"{stray_dir}/location", "w") as fd: + fd.write("should not be listed") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + ids = {repo["id"] for repo in result_json["repositories"]} + assert ids == {repo_id} + + +def test_known_repos_multiple_sorted_by_id(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id1 = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + # a second, independent repository in a different location + second_repo = str(archiver.repository_path) + "-second" + ret, output = exec_cmd( + f"--repo={second_repo}", + "repo-create", + RK_ENCRYPTION, + archiver=archiver.archiver, + fork=archiver.FORK_DEFAULT, + exe=archiver.EXE, + ) + assert ret == 0, output + ret, output = exec_cmd( + f"--repo={second_repo}", + "repo-info", + "--json", + archiver=archiver.archiver, + fork=archiver.FORK_DEFAULT, + exe=archiver.EXE, + ) + assert ret == 0, output + info2 = json.loads(output)["repository"] + repo_id2 = info2["id"] + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + assert set(repos_by_id) == {repo_id1, repo_id2} + assert repos_by_id[repo_id2]["location"] == info2["location"] + # sorted by repository id + assert [repo["id"] for repo in result_json["repositories"]] == sorted([repo_id1, repo_id2]) + + +def test_known_repos_scans_legacy_v1_security_dir(archivers, request): + """A v1-style security entry (legacy $BORG_CONFIG_DIR/security layout) is listed too.""" + archiver = request.getfixturevalue(archivers) + # the archiver fixture isolates BORG_BASE_DIR to a per-test tmp dir, so BORG_CONFIG_DIR + # (unset here) resolves below it via legacy.fs.get_config_dir() - safe to plant a fake entry in. + from borg.legacy.fs import get_config_dir as get_config_dir_legacy + + legacy_repo_id = "a" * 64 # any syntactically valid 64-hex id; this test never opens a repo + legacy_security_dir = os.path.join(get_config_dir_legacy(create=False), "security", legacy_repo_id) + os.makedirs(legacy_security_dir, exist_ok=True) + with open(os.path.join(legacy_security_dir, "location"), "w") as fd: + fd.write("/some/v1/repo/path") + with open(os.path.join(legacy_security_dir, "key-type"), "w") as fd: + fd.write("3") # KeyType.REPO (borg 1.x repokey, legacy AES-CTR) + + result = cmd(archiver, "known-repos") + assert f"Repository ID: {legacy_repo_id}" in result + assert "Layout: legacy" in result + assert "Local cache: no" in result + assert "Security info: yes (aes256-ctr, sha256 (legacy))" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + legacy_repo = repos_by_id[legacy_repo_id] + assert legacy_repo["layout"] == "legacy" + assert legacy_repo["location"] == "/some/v1/repo/path" + assert legacy_repo["key_type"] == "aes256-ctr, sha256 (legacy)" + assert legacy_repo["has_cache"] is False + + +def test_known_repos_does_not_write_cachedir_tag(archivers, request): + """Running known-repos on an otherwise-empty cache dir must not plant CACHEDIR.TAG there. + + get_cache_dir(create=True) (the default) writes a CACHEDIR.TAG marker file as a side effect; + known-repos uses create=False specifically to avoid that, since it is meant to be read-only. + """ + archiver = request.getfixturevalue(archivers) + cache_root = get_cache_dir(create=False) + cachedir_tag = os.path.join(cache_root, "CACHEDIR.TAG") + assert not os.path.exists(cachedir_tag) # the archiver fixture's cache dir starts empty + + result = cmd(archiver, "known-repos") + assert "No locally known repositories." in result + assert not os.path.exists(cachedir_tag) + + +@requires_symlinks +def test_known_repos_does_not_follow_symlinked_cache_entry(archivers, request): + """A cache_root/ entry that is a symlink (not a real directory) must be skipped, not followed. + + Borg itself always creates a plain directory there; a symlink can only come from something else + (deliberate relocation, or an attempt to redirect the read elsewhere) and must not be treated as + a cache. + """ + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", size=1024) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input") + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + cache_root = get_cache_dir(create=False) + assert cache_root.startswith(str(archiver.cache_path) + os.sep) or cache_root == str(archiver.cache_path) + real_cache_dir = os.path.join(cache_root, repo_id) + moved_cache_dir = real_cache_dir + "-moved" + os.rename(real_cache_dir, moved_cache_dir) + os.symlink(moved_cache_dir, real_cache_dir) + + result = cmd(archiver, "known-repos") + assert "Local cache: no" in result + assert "Security info: yes (aes256-ocb, sha256)" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["has_cache"] is False + assert repo["cache_size"] is None + + +@requires_symlinks +def test_known_repos_does_not_follow_symlinked_security_entry(archivers, request): + """A security_root/ entry that is a symlink must be skipped too, same reasoning as the cache one.""" + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + # repo-create also creates a local cache; remove it first so this id has *only* security info, + # all of it reachable only through the symlink this test is about to plant. + cache_path = get_cache_dir(repo_id, create=False) + assert cache_path.startswith(str(archiver.cache_path) + os.sep) + shutil.rmtree(cache_path, ignore_errors=True) + + security_root = get_security_dir(create=False) + real_security_dir = os.path.join(security_root, repo_id) + moved_security_dir = real_security_dir + "-moved" + os.rename(real_security_dir, moved_security_dir) + os.symlink(moved_security_dir, real_security_dir) + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + ids = {repo["id"] for repo in result_json["repositories"]} + # the id only had security info (via the now-symlinked dir, which must be ignored) and no real + # cache directory, so it must not be reported as a known repository at all. + assert repo_id not in ids + + +@requires_symlinks +def test_known_repos_cache_only_discovery_does_not_leak_symlinked_security_dir(archivers, request): + """A real cache dir plus a symlinked (attacker-controlled-content) security dir for the same id + + must not have that security content read and reported: the id is discovered via its real cache, + but the security_root/ path is a symlink that scan_security_root would reject on its own - + discovering the id through the cache must not grant an implicit pass to read it anyway. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + security_root = get_security_dir(create=False) + real_security_dir = os.path.join(security_root, repo_id) + fake_dir = os.path.join(security_root, "fake-target-for-" + repo_id) + os.makedirs(fake_dir, exist_ok=True) + with open(os.path.join(fake_dir, "location"), "w") as fd: + fd.write("/not/the/real/location") + shutil.rmtree(real_security_dir, ignore_errors=True) + os.symlink(fake_dir, real_security_dir) + + result = cmd(archiver, "known-repos") + assert f"Repository ID: {repo_id}" in result + assert "Local cache: yes" in result + assert "/not/the/real/location" not in result + assert "Security info: no" in result + assert "Note: a local cache exists, but no security info was found" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = next(r for r in result_json["repositories"] if r["id"] == repo_id) + assert repo["location"] is None + assert repo["has_security_info"] is False + + +@requires_symlinks +def test_known_repos_does_not_follow_symlinked_location_file(archivers, request): + """A location file that is itself a symlink (inside an otherwise real, non-symlinked security dir) + + must not be followed either - rejecting a symlinked repo-id *directory* is not enough if a + symlinked *file* inside a real directory can still redirect the read. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + security_dir = os.path.join(get_security_dir(create=False), repo_id) + fake_location_file = os.path.join(security_dir, "fake-location.txt") + with open(fake_location_file, "w") as fd: + fd.write("/attacker/leaf/path") + real_location_file = os.path.join(security_dir, "location") + os.remove(real_location_file) + os.symlink(fake_location_file, real_location_file) + + result = cmd(archiver, "known-repos") + assert "/attacker/leaf/path" not in result + assert "Location: (unknown)" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["location"] is None + + +@pytest.mark.skipif(os.name == "nt", reason="Windows rejects a newline in a filename at creation time") +def test_known_repos_rejects_repo_id_with_trailing_newline(archivers, request): + """A directory name of 64 hex digits followed by a newline must not match the repo-id pattern. + + re.match with a plain trailing '$' anchor also matches just before a final newline; a directory + actually named "<64 hex>\\n" (an unusual but possible local name) must not be treated as a repo. + """ + archiver = request.getfixturevalue(archivers) + security_root = get_security_dir(create=False) + bad_name = "a" * 64 + "\n" + bad_dir = os.path.join(security_root, bad_name) + os.makedirs(bad_dir, exist_ok=True) + with open(os.path.join(bad_dir, "location"), "w") as fd: + fd.write("/should/not/be/listed") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + assert result_json["repositories"] == [] + + +def test_known_repos_scans_legacy_cache_root_when_different_from_borg2s(request, monkeypatch): + """A v1-only cache (in legacy.fs.get_cache_dir()'s root, not borg2's) must still be listed. + + On platforms where platformdirs (borg2) and the old home-grown layout (legacy.fs) resolve to + different default cache roots (e.g. macOS: "~/Library/Caches/borg" vs "~/.cache/borg"), a v1 + repository's cache lives only in the legacy root. The test fixture sets BORG_BASE_DIR, which + makes both roots resolve to the same place in practice - monkeypatch legacy.fs.get_cache_dir + directly to simulate the platforms where they differ, without depending on the real platform + this test happens to run on. + + Local only (not parametrized over archivers/binary_archiver): the monkeypatch only takes + effect in this pytest process, so it would silently not apply if run against a forked + borg.exe subprocess (binary_archiver) - there is no env var that makes legacy.fs and + helpers.fs resolve to different cache roots to simulate this instead. + """ + archiver = request.getfixturevalue("archiver") + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + # move the real cache dir into a separate "legacy" root that differs from the borg2 cache root. + borg2_cache_root = get_cache_dir(create=False) + real_cache_dir = os.path.join(borg2_cache_root, repo_id) + legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") + os.makedirs(legacy_cache_root, exist_ok=True) + os.rename(real_cache_dir, os.path.join(legacy_cache_root, repo_id)) + + import borg.archiver.known_repos_cmd as known_repos_cmd + + monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) + + result = cmd(archiver, "known-repos") + assert f"Repository ID: {repo_id}" in result + assert "Local cache: yes" in result + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["id"] == repo_id + assert repo["has_cache"] is True + assert repo["cache_size"] > 0 + + +def test_known_repos_warns_and_exits_nonzero_on_oversized_key_type(archivers, request): + """A key-type file too large to interpret must log a warning AND affect the exit code. + + logger.warning() alone does not do that - this command must additionally call + self.print_warning() (see Archiver.print_warning / EXIT_WARNING) so that a script checking the + exit status also learns that the audit was incomplete, not just a human reading stderr. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + key_type_path = os.path.join(get_security_dir(repo_id, create=False), "key-type") + with open(key_type_path, "w") as fd: + fd.write("x" * 5000) # over the 4096 byte cap in known_repos_cmd._Inspector.read_subfile + + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + assert f"Repository ID: {repo_id}" in result + assert "Security info: yes (key type unknown)" in result + + +def test_known_repos_warns_and_exits_nonzero_on_non_utf8_location(archivers, request): + """A location file that is not valid UTF-8 must be treated like read_subfile returned None: + + warned, counted incomplete, and (since key-type is still readable) reported as + "Security info: yes" without a location, not silently substituted or crashed on. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + location_path = os.path.join(get_security_dir(repo_id, create=False), "location") + with open(location_path, "wb") as fd: + fd.write(b"\xff\xfe\x00not valid utf-8") + + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + assert f"Repository ID: {repo_id}" in result + assert "Location: (unknown)" in result + assert "Security info: yes" in result + + +def test_known_repos_escapes_control_characters_in_location(archivers, request): + """A location value containing control characters (e.g. from a crafted/corrupted file) must + + be escaped before printing, not written to the terminal raw - _printable() exists precisely + to stop a stored location/key-type value from injecting terminal control sequences. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + location_path = os.path.join(get_security_dir(repo_id, create=False), "location") + with open(location_path, "w") as fd: + fd.write("/tmp/evil\x1b[31mred\x1b[0m") + + result = cmd(archiver, "known-repos") + assert f"Repository ID: {repo_id}" in result + # the raw ESC byte must not appear unescaped; _printable() renders it as a Python repr + # escape (e.g. "\x1b") instead. + assert "\x1b" not in result + assert "\\x1b" in result + + +def test_known_repos_missing_key_type_file_is_not_a_warning(archivers, request): + """A repo with no key-type file at all (location-only) must not trigger the incomplete-scan warning. + + This is the ordinary "not found" case (see test_known_repos_location_only_counts_as_security_info) + - distinct from "found but could not be read", which does warn. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + security_path = get_security_dir(repo_id, create=False) + os.remove(os.path.join(security_path, "key-type")) + + result = cmd(archiver, "known-repos") # default exit_code=0 in cmd(); fails if exit code is nonzero + assert "could not be fully completed" not in result + assert f"Repository ID: {repo_id}" in result + + +@requires_chmod_permissions +def test_known_repos_unlistable_cache_subdir_reports_partial_size_not_missing(archivers, request): + """A cache dir that exists, but has a subdirectory that can't be listed, is NOT reported as missing. + + Unlike an unreadable security file (treated the same as absent, see other tests in this file), + an unlistable part of an otherwise-present cache directory still counts as "Local cache: yes" - + just with an incomplete (here: smaller than the real) size, plus a warning and nonzero exit code. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + cache_dir = get_cache_dir(repo_id, create=False) + # baseline size of whatever repo-create left (config/README/...), before the unlistable subtree. + baseline = 0 + for dirpath, dirnames, filenames in os.walk(cache_dir): + for name in filenames: + baseline += os.path.getsize(os.path.join(dirpath, name)) + + unlistable_subdir = os.path.join(cache_dir, "unlistable") + os.makedirs(unlistable_subdir, exist_ok=True) + with open(os.path.join(unlistable_subdir, "somefile"), "w") as fd: + fd.write("x" * 1000) + os.chmod(unlistable_subdir, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + assert "Local cache: yes" in result # not reported as missing, unlike the security-file case + # the unlistable subdir's 1000 bytes are not counted - this is the partial-count behavior + # the epilog describes, not the exact size a healthy scan would report. + assert f"Local cache: yes, {format_file_size(baseline)}" in result + assert "1.00 kB" not in result and "1000 B" not in result + + # --json is not checked here: in the in-process (non-fork) test harness, stdout and stderr + # are the same stream (see exec_cmd()), so the warning line above would land inside what is + # otherwise supposed to be pure JSON output. That collision is a test-harness property that + # also exists for every other command with a --json mode, not something specific to this + # command, and real (forked, non-test) usage keeps the warning on stderr and --json's output + # on stdout separately. + finally: + os.chmod(unlistable_subdir, 0o755) + + +@requires_chmod_permissions +def test_known_repos_unreadable_borg2_cache_does_not_fall_back_to_legacy(request, monkeypatch): + """An unlistable borg2 cache must be reported as 'no cache', and must not publish the legacy + cache's size/mtime for the same id. + + open_cache_entry returns 'unreadable' when the directory is confirmed real but its config + could not be inspected; the command then treats that as "no cache" rather than falling back + to the other root, since the borg2 side's actual state was never established. Mode 0o400 + (readable, not searchable) is used rather than 0o000 so the repo-id directory itself still + opens - only the later stat of its "config" child fails with EACCES, which is the branch + this test targets (0o000 would instead fail at the directory open itself). + """ + archiver = request.getfixturevalue("archiver") + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + borg2_cache_root = get_cache_dir(create=False) + real_cache_dir = os.path.join(borg2_cache_root, repo_id) + legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") + os.makedirs(os.path.join(legacy_cache_root, repo_id), exist_ok=True) + with open(os.path.join(legacy_cache_root, repo_id, "config"), "w") as fd: + fd.write("not-the-borg2-cache") + with open(os.path.join(legacy_cache_root, repo_id, "decoy"), "w") as fd: + fd.write("x" * 5000) + + import borg.archiver.known_repos_cmd as known_repos_cmd + + monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) + + os.chmod(real_cache_dir, 0o400) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + # the borg2 cache's own state is unknown, so this must not claim "yes" (an unverified + # existence claim) or "no" (an unverified absence claim), and must not silently report + # the legacy root's cache either. + assert "Local cache: unknown" in result + # the 5000-byte decoy lives only in the legacy root; "Local cache: unknown" above + # already proves the legacy cache was not substituted (it would have had to report a + # size to do that), so no separate size assertion is needed. --json is not checked + # here: in the in-process (non-fork) test harness, stdout and stderr are the same + # stream, so the warning line above would land inside what is otherwise supposed to + # be pure JSON output (same test-harness property noted elsewhere in this file). + finally: + os.chmod(real_cache_dir, 0o755) + + +@requires_chmod_permissions +def test_known_repos_unsearchable_cache_root_with_nonexistent_child_reports_no_cache(request, monkeypatch): + """An unsearchable cache ROOT (not just one repo-id subdirectory within it) must not make a + + repo-id that never had a cache there look like it has one. When the root itself can't be + opened, cache_root_d is None and kind comes from cache_root_kind ("unreadable") - this + path never reaches open_cache_entry(), since there is no cache_root_d to call it with. + That "unreadable" result must not be treated as has_cache=True, and must not fall back to + a legacy cache either - the borg2 side's existence was never confirmed absent. This is + distinct from the repo-id-directory-level unreadable-cache test above, which makes an + existing repo-id directory (not its parent root) unreadable. + """ + archiver = request.getfixturevalue("archiver") + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + borg2_cache_root = get_cache_dir(create=False) + # remove the normal cache dir for this id entirely - only the unsearchable empty root remains. + shutil.rmtree(os.path.join(borg2_cache_root, repo_id), ignore_errors=True) + + legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") + os.makedirs(os.path.join(legacy_cache_root, repo_id), exist_ok=True) + with open(os.path.join(legacy_cache_root, repo_id, "config"), "w") as fd: + fd.write("not-the-borg2-cache") + + import borg.archiver.known_repos_cmd as known_repos_cmd + + monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) + + os.chmod(borg2_cache_root, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + # this repo-id was only discovered via the (unreadable) legacy cache root here - its id + # comes from scan_cache_root() on the legacy root reporting it present, while the borg2 + # root's state for it could never be established (root itself unsearchable). The borg2 + # side being unknown must not fabricate "yes", and must not silently claim a confirmed + # "no" either - it surfaces as unknown. --json is not checked (see the test-harness + # note on the previous test in this file). + assert "Local cache: unknown" in result + finally: + os.chmod(borg2_cache_root, 0o755) + + +@requires_chmod_permissions +def test_known_repos_unopenable_security_entry_is_not_dropped(archivers, request): + """A repo-id directory under the security root that list_names() can see, but that this + + command cannot open (EACCES), must still appear in the result - with unknown fields - not + silently disappear from the inventory just because its own inspection failed. Before the + fix for this, scan_security_root()'s `if entry is None: continue` treated "confirmed absent + (missing/symlink)" and "inspection failed (EACCES)" identically, so a name list_names() had + already returned could vanish without a trace in the output - only the summary warning and + exit code hinted anything was wrong, with no way to tell which repository was affected. + """ + archiver = request.getfixturevalue(archivers) + security_root = get_security_dir(create=False) + repo_id = "d" * 64 + security_path = os.path.join(security_root, repo_id) + os.makedirs(security_path, exist_ok=True) + with open(os.path.join(security_path, "location"), "w") as fd: + fd.write("/should/not/be/readable") + os.chmod(security_path, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + # the repository must still be LISTED - not dropped - with its security/layout fields + # reported as unknown rather than a false "no" or a false "yes". The cache root itself + # was searched successfully and genuinely has no entry for this id, so "Local cache: + # no" here is a real, confirmed answer, not a false negative - this test is only about + # the security-side unreadable entry not causing the repo-id to disappear. --json is + # not checked here (see the in-process stdout/stderr test-harness note elsewhere in + # this file). + assert f"Repository ID: {repo_id}" in result + assert "Layout: unknown" in result + assert "Security info: unknown" in result + assert "Local cache: no" in result + finally: + os.chmod(security_path, 0o755) + + +@requires_chmod_permissions +def test_known_repos_unopenable_cache_entry_is_not_dropped(archivers, request): + """Same as test_known_repos_unopenable_security_entry_is_not_dropped, but for a repo-id + + directory under the cache root that this command cannot open at all (not just whose + "config" child can't be stat'd - see test_known_repos_unreadable_borg2_cache_does_not_fall_back_to_legacy + for that case). scan_cache_root()'s discovery pass must not drop this name either. + """ + archiver = request.getfixturevalue(archivers) + cache_root = get_cache_dir(create=False) + repo_id = "e" * 64 + cache_path = os.path.join(cache_root, repo_id) + os.makedirs(cache_path, exist_ok=True) + with open(os.path.join(cache_path, "config"), "w") as fd: + fd.write("irrelevant") + os.chmod(cache_path, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + assert f"Repository ID: {repo_id}" in result + assert "Local cache: unknown" in result + # --json is not checked here (see the in-process stdout/stderr test-harness note + # elsewhere in this file). + finally: + os.chmod(cache_path, 0o755) + + +@requires_chmod_permissions +def test_known_repos_unopenable_cache_entry_with_security_confirmed_absent(archivers, request): + """Same setup as test_known_repos_unopenable_cache_entry_is_not_dropped, but with both + + security roots present and empty (confirmed searched, not just absent by default): + security info must be reported as confirmed absent ("no"), not "unknown". Before the + fix for this, this id fell into do_known_repos()'s catch-all else branch (reached only + via the unreadable-cache discovery path, not borg2_ids/legacy_ids/*_cache_ids), which + unconditionally set has_security_info to None even though the security side had + already been fully and successfully searched. + """ + archiver = request.getfixturevalue(archivers) + # touch both security roots so they exist and are searchable (still empty for this id). + get_security_dir(create=True) + from borg.legacy.fs import get_security_dir as get_security_dir_legacy + + get_security_dir_legacy(create=True) + + cache_root = get_cache_dir(create=False) + repo_id = "f" * 64 + cache_path = os.path.join(cache_root, repo_id) + os.makedirs(cache_path, exist_ok=True) + with open(os.path.join(cache_path, "config"), "w") as fd: + fd.write("irrelevant") + os.chmod(cache_path, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert f"Repository ID: {repo_id}" in result + assert "Security info: no" in result + assert "Local cache: unknown" in result + finally: + os.chmod(cache_path, 0o755) + + +@requires_chmod_permissions +def test_known_repos_unsearchable_security_parent_warns(archivers, request): + """A security root whose parent cannot be searched must warn, not look empty. + + Path.is_dir() on Python 3.14 returns False for this case without raising. A naive + "if not is_dir(): treat as absent" check would therefore exit 0 on an unreadable parent. + The actual code opens the root via os_open() (not Path.is_dir()), which raises EACCES here + and is counted as incomplete via _warn(). + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + security_root = get_security_dir(create=False) + parent = os.path.dirname(security_root) + # only chmod a parent that is inside this test's isolated BORG_BASE_DIR + assert parent.startswith(os.environ["BORG_BASE_DIR"] + os.sep) + os.chmod(parent, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + # the repo-create above also left a local cache, so this id is still discovered (via + # the cache root, which is unaffected by chmod'ing the security root's parent) - but + # with the ENTIRE security root unsearchable, "no security entry for this id" can + # never be confirmed; it must surface as unknown, not a false "no". + assert f"Repository ID: {repo_id}" in result + assert "Security info: unknown" in result + finally: + os.chmod(parent, 0o755) + + +@requires_chmod_permissions +def test_known_repos_unreadable_legacy_cache_root_with_confirmed_absent_borg2_cache(request, monkeypatch): + """When the current (borg2) cache is confirmed absent for an id, but the legacy cache + + ROOT ITSELF cannot even be opened (not just a specific entry within it - see + test_known_repos_unreadable_borg2_cache_does_not_fall_back_to_legacy for that case), this + command must not claim "no cache": the legacy side was never actually searched, so the + true answer is unknown. Before the fix for this, the "absent" fallback condition only + checked `legacy_cache_root_d is not None`, which is also true/false independent of whether + the legacy root could be *opened* - if it couldn't be opened, kind simply stayed "absent" + from the already-confirmed-absent borg2 side, silently skipping the (failed) legacy check + instead of surfacing that failure. + """ + archiver = request.getfixturevalue("archiver") + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + # this id has security info (from repo-create) but no borg2 cache entry - delete it so the + # borg2 cache side is confirmed absent, forcing the legacy-cache fallback path. + borg2_cache_root = get_cache_dir(create=False) + shutil.rmtree(os.path.join(borg2_cache_root, repo_id), ignore_errors=True) + + legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") + os.makedirs(legacy_cache_root, exist_ok=True) + + import borg.archiver.known_repos_cmd as known_repos_cmd + + monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) + + os.chmod(legacy_cache_root, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + assert f"Repository ID: {repo_id}" in result + # borg2 cache: confirmed absent. legacy cache ROOT: could not even be opened - the + # legacy side was never searched, so the combined answer must be unknown, not "no". + assert "Local cache: unknown" in result + finally: + os.chmod(legacy_cache_root, 0o755) + + +def test_known_repos_security_root_listing_failure_does_not_confirm_cache_only_id_absent(request, monkeypatch): + """A security root that opens fine but fails to LIST (e.g. EIO) must not let a + + cache-only id be reported as having confirmed-absent security info - scan_security_root() + cannot have seen every entry under a root it failed to list, so "this id has no security + entry" is not something it can actually assert, even though no individual repo-id + directory's open failed. Before the fix for this, only root-OPEN failure and per-id EACCES + were tracked (security_root_kind / security_unreadable_ids) - a root that opened + successfully and then failed during list_names() itself fell through neither check, so + security_fully_searched stayed True and a cache-only id got a false "no". + """ + archiver = request.getfixturevalue("archiver") + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + # this id has a cache (from repo-create) but no security entry for it specifically is + # required - delete the security entry so this id is discovered ONLY via the cache root, + # exercising the borg2_cache_ids branch in do_known_repos(). + security_path = get_security_dir(repo_id, create=False) + shutil.rmtree(security_path, ignore_errors=True) + + import borg.archiver.known_repos_cmd as known_repos_cmd + + orig_list_names = known_repos_cmd._Inspector.list_names + security_root = get_security_dir(create=False) + + def failing_list_names(self, d, display): + # simulate a successful open followed by a listing failure (e.g. EIO) specifically for + # the (already-opened) security root, leaving every other list_names() call (cache + # root, individual repo-id directories) unaffected. + if display == security_root: + raise OSError(5, "Input/output error") # EIO, caught the same way a real one is + return orig_list_names(self, d, display) + + def patched_list_names(self, d, display): + try: + return failing_list_names(self, d, display) + except OSError as exc: + self._warn("known-repos: could not list %s, results may be incomplete: %s", display, exc) + return [] + + monkeypatch.setattr(known_repos_cmd._Inspector, "list_names", patched_list_names) + + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + assert f"Repository ID: {repo_id}" in result + # the security root's own listing failed - "no security entry for this id" was never + # actually confirmed, so this must be unknown, not a false "no". + assert "Security info: unknown" in result + # the cache side was unaffected by this failure and should still be confirmed present. + assert "Local cache: yes" in result + + +@pytest.mark.skipif(os.name == "nt", reason="FIFOs via os.mkfifo are not available on Windows") +def test_known_repos_rejects_fifo_location_without_hanging(archivers, request): + """A stationary FIFO named 'location' (no writer) must be rejected quickly, not hang. + + Opening a FIFO read-only blocks until a writer connects; read_subfile must use O_NONBLOCK + on the dir_fd-relative open so this command never stalls on one. + """ + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + security_path = get_security_dir(repo_id, create=False) + location_path = os.path.join(security_path, "location") + os.remove(location_path) + os.mkfifo(location_path) + + # cmd() runs in-process (no fork) for the "archiver" param, so a regression of O_NONBLOCK + # here would hang this test rather than fail it cleanly. + result = cmd(archiver, "known-repos") + assert f"Repository ID: {repo_id}" in result + assert "Location: (unknown)" in result + + +def test_known_repos_security_only_repo_with_oversized_location_is_still_listed(archivers, request): + """A repo whose ONLY local state is an oversized (unreadable) location file must still be + + listed with has_security_info True (the file's presence IS confirmed via stat, independent + of whether its content could be decoded) - not disappear, and not be downgraded to "no" + just because the content itself could not be read. + """ + archiver = request.getfixturevalue(archivers) + security_root = get_security_dir(create=False) + repo_id = "c" * 64 + security_path = os.path.join(security_root, repo_id) + os.makedirs(security_path, exist_ok=True) + with open(os.path.join(security_path, "location"), "w") as fd: + fd.write("x" * 5000) # over the 4096 byte cap + + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be fully completed" in result + assert f"Repository ID: {repo_id}" in result + assert "Location: (unknown)" in result + # the oversized location file is this repo's ONLY security state; it was confirmed PRESENT + # via stat (independent of the content read that then failed), so this is "Security info: + # yes" with an unknown key type, not "no" - "no" would be a false claim of absence for a + # file this command already knows exists. + assert "Security info: yes (key type unknown)" in result + + # --json is not checked here: in the in-process (non-fork) test harness, stdout and stderr + # are the same stream, so the warning line above would land inside what is otherwise + # supposed to be pure JSON output (same test-harness property noted elsewhere in this file). + + +def test_known_repos_deep_cache_tree_does_not_hit_recursion_limit(archivers, request): + """dir_size must use an explicit stack, not Python recursion, so a cache directory deeper + + than sys.getrecursionlimit() does not abort the whole command with RecursionError. + + dir_size() holds one fd open per currently-descended level (not per sibling), so a tree + this deep needs that many simultaneously open fds - more than the default soft limit on + macOS (256) or many Linux distros (1024). Raise the soft limit for this test only (never + above the hard limit) rather than reducing the depth, since the depth itself is what + exercises sys.getrecursionlimit(); skip on platforms without an fd-based soft/hard limit + (e.g. Windows, which uses the Path fallback and does not hold one fd per level). + """ + if resource is None or not hasattr(resource, "RLIMIT_NOFILE"): + pytest.skip("fd-based open-files limit (resource.RLIMIT_NOFILE) not available on this platform") + depth = 1500 # beyond sys.getrecursionlimit()'s default (~1000) + needed_fds = depth + 64 # +margin for fds pytest/borg itself already holds open + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if hard != resource.RLIM_INFINITY and hard < needed_fds: + pytest.skip(f"hard open-files limit ({hard}) is below what this depth needs ({needed_fds})") + if soft < needed_fds: + resource.setrlimit(resource.RLIMIT_NOFILE, (needed_fds, hard)) + request.addfinalizer(lambda: resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))) + + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + + cache_dir = get_cache_dir(repo_id, create=False) + # build via relative single-component mkdir+chdir steps, not one long os.path.join'd + # absolute path, to avoid hitting the OS's own path-length limit (ENAMETOOLONG) long + # before reaching a depth that would actually exercise Python's recursion limit. + previous_cwd = os.getcwd() + try: + os.chdir(cache_dir) + for d in range(depth): + os.mkdir(f"d{d}") + os.chdir(f"d{d}") + with open("leaf", "w") as fd: + fd.write("x" * 7) + finally: + os.chdir(previous_cwd) + + result = cmd(archiver, "known-repos") + assert f"Repository ID: {repo_id}" in result + assert "Local cache: yes" in result + + +@pytest.mark.skipif(os.name == "nt", reason="test assumes a real fd; Windows always uses the Path fallback") +def test_dir_size_closes_every_handle_when_list_names_raises_base_exception(tmpdir): + """dir_size() must close root (and any already-opened subdirectory) even when list_names() + + raises something that is not an OSError (e.g. KeyboardInterrupt) - including on the very + first call for root, and on the first call for a freshly opened subdirectory, before either + one has been appended to/left on the stack by a completed iteration. A fd leaked here is a + real resource leak, not just a style concern: this is a focused unit test against + _Inspector directly (not the CLI), since the leak is only observable via the raw fd. + """ + from borg.archiver.known_repos_cmd import _Inspector + from pathlib import Path + + root = Path(tmpdir) # open_root() is always called with a Path in production code + sub = root / "sub" + sub.mkdir() + (sub / "f.txt").write_bytes(b"x" * 10) + + inspector = _Inspector() + root_fd = inspector.open_root(root, str(root)) + assert isinstance(root_fd, int), "test assumes a real fd, not the Windows Path fallback" + + captured = {} + orig_open_subdir = inspector.open_subdir + + def capturing_open_subdir(parent, name, display): + fd = orig_open_subdir(parent, name, display) + captured["sub_fd"] = fd + return fd + + inspector.open_subdir = capturing_open_subdir + + orig_list_names = inspector.list_names + calls = {"n": 0} + + def flaky_list_names(d, display): + calls["n"] += 1 + if calls["n"] == 2: # the second call is for "sub", opened by the loop below + raise KeyboardInterrupt("simulated interrupt") + return orig_list_names(d, display) + + inspector.list_names = flaky_list_names + + with pytest.raises(KeyboardInterrupt): + inspector.dir_size(root_fd, root) + + with pytest.raises(OSError): + os.fstat(root_fd) + sub_fd = captured.get("sub_fd") + assert isinstance(sub_fd, int) + with pytest.raises(OSError): + os.fstat(sub_fd) + + +@pytest.mark.skipif(os.name == "nt", reason="test assumes a real fd; Windows always uses the Path fallback") +def test_dir_size_closes_root_when_its_own_list_names_raises_base_exception(tmpdir): + """Same leak class as above, but for root's own first list_names() call specifically - + + the one at the very start of dir_size(), before the while loop has run at all. + """ + from borg.archiver.known_repos_cmd import _Inspector + from pathlib import Path + + root = Path(tmpdir) # open_root() is always called with a Path in production code + inspector = _Inspector() + root_fd = inspector.open_root(root, str(root)) + assert isinstance(root_fd, int), "test assumes a real fd, not the Windows Path fallback" + + def raising_list_names(d, display): + raise KeyboardInterrupt("simulated interrupt") + + inspector.list_names = raising_list_names + + with pytest.raises(KeyboardInterrupt): + inspector.dir_size(root_fd, root) + + with pytest.raises(OSError): + os.fstat(root_fd) From c0c493467bcafbf1c100b4136c081fae52ed570f Mon Sep 17 00:00:00 2001 From: Kazuki Kanda <96773362+KazukiKandaKK@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:30:36 +0900 Subject: [PATCH 2/5] known-repos: simplify after maintainer review, drop leftover defenses Rewrite the scanner around ordinary OS file access instead of extra symlink/size defenses, collapse duplicated scan/table/test logic, and surface real inspection failures (ENOTDIR, unparseable cache config) instead of treating them as absence. Also forward create=False through legacy get_keys_dir(), refresh the usage example, keep tests to the remaining behavioral surface, and shorten a few overly verbose test docstrings. --- docs/man/borg-known-repos.1 | 171 --- docs/usage/known-repos.rst | 19 +- docs/usage/known-repos.rst.inc | 132 -- src/borg/archiver/known_repos_cmd.py | 906 ++++---------- src/borg/legacy/fs.py | 14 +- .../archiver/known_repos_cmd_test.py | 1098 ++++------------- src/borg/testsuite/legacy_fs_test.py | 31 + 7 files changed, 543 insertions(+), 1828 deletions(-) delete mode 100644 docs/man/borg-known-repos.1 delete mode 100644 docs/usage/known-repos.rst.inc diff --git a/docs/man/borg-known-repos.1 b/docs/man/borg-known-repos.1 deleted file mode 100644 index fde2eb8989..0000000000 --- a/docs/man/borg-known-repos.1 +++ /dev/null @@ -1,171 +0,0 @@ -.\" Man page generated from reStructuredText -.\" by the Docutils 0.23 manpage writer. -. -. -.nr rst2man-indent-level 0 -. -.de1 rstReportMargin -\\$1 \\n[an-margin] -level \\n[rst2man-indent-level] -level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] -- -\\n[rst2man-indent0] -\\n[rst2man-indent1] -\\n[rst2man-indent2] -.. -.de1 INDENT -.\" .rstReportMargin pre: -. RS \\$1 -. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] -. nr rst2man-indent-level +1 -.\" .rstReportMargin post: -.. -.de UNINDENT -. RE -.\" indent \\n[an-margin] -.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] -.nr rst2man-indent-level -1 -.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] -.in \\n[rst2man-indent\\n[rst2man-indent-level]]u -.. -.TH "borg-known-repos" "1" "2026-10-10" "" "borg backup tool" -.SH Name -borg-known-repos \- List repositories Borg has local state for (security info and/or cache). -.SH SYNOPSIS -.sp -borg [common options] known\-repos [options] -.SH DESCRIPTION -.sp -This command lists the repositories that this local Borg installation currently -has state for: security information (key type, last known location) and/or a -local repository cache. -.sp -It inspects only local state \- the security directory and the cache directory, -each checked both at its current (borg2) default location and, separately, at -the legacy (borg 1.x) default location those same kinds of state used before the -borg2 base\-dir rewrite (see \fB$BORG_SECURITY_DIR\fP/\fB$BORG_CACHE_DIR\fP in -\fBborg help environment\fP for the exact resolution rules, including the platforms -where the two locations differ by default, e.g. macOS). It does not access any -repository, locally or remotely, so a repository that is currently unreachable -(e.g. a disconnected external disk or an unresponsive remote host) will still be -listed just fine. This also means \-r/\-\-repo has no effect on this command: it -always lists every locally known repository. -.sp -\(dqLayout\(dq reflects which state directory tree the repository was discovered in \- -\fBborg2\fP or \fBlegacy\fP \- not the repository\(aqs actual on\-disk format version (see -\fBborg repo\-info\fP\(aqs \(dqRepository version\(dq for that; this command never opens the -repository, so it cannot report that field at all). The borg2 security entry is -checked first, then the legacy (borg 1.x) security entry, then \- only when -neither security entry could be confirmed \- whichever cache root matched (borg2 -cache checked before the legacy cache). \(dqunknown\(dq means neither a security entry -nor a cache entry could be confirmed for this id \- it was listed as a directory -name, but this command could not establish which layout (if either) it belongs -to. If \fB$BORG_SECURITY_DIR\fP/\fB$BORG_CACHE_DIR\fP make the borg2 and legacy -locations resolve to the same path, only the borg2 scan runs and a legacy (v1) -repository there is reported with layout \fBborg2\fP\&. -.sp -This is useful for auditing which repositories this installation has local state for -(and, for repository\-encrypted modes, which crypto suite each one uses), finding -repositories with a local cache that may be taking up unwanted disk space, and general -housekeeping of stale local Borg state. -.sp -The reported cache size is the size of all regular files below the repository\(aqs -cache directory (symlinks are not followed and not counted) \- for a current (borg2) -repository that is the files cache plus a small amount of metadata (\fBconfig\fP, -\fBREADME\fP), not an index of the repository\(aqs chunks; a legacy (borg 1.x) cache -directory can be considerably larger, since it also holds a local chunks index. -A repository\-id directory that is a symlink (Borg itself always creates a plain -directory) is skipped, and a \fBlocation\fP or \fBkey\-type\fP file that is a symlink -is treated the same as missing; neither is followed. -.sp -\(dqCache last written\(dq is the modification time of the cache\(aqs own config file, as -a best\-effort proxy for when a cache\-using Borg command last ran against this -repository. It is not a precise \(dqlast backup\(dq or \(dqlast access\(dq time: not every -Borg command updates the cache, and a repository without a local cache at all -will not show one. -.sp -This command does not list or export any key material: the keys dir -(\fB$BORG_KEYS_DIR\fP) is not scanned. -.sp -\(dqSecurity info\(dq and \(dqLocal cache\(dq are each one of three states, not a plain -yes/no: \(dqyes\(dq (confirmed present \- for security info, a \fBlocation\fP and/or -\fBkey\-type\fP file was confirmed to exist there, independent of whether its -\fIcontent\fP could be read; for cache, a real, non\-symlinked cache directory with a -\fBconfig\fP file was confirmed to exist), \(dqno\(dq (confirmed absent \- this command -actually searched the relevant location(s) and found nothing there \- for the -cache specifically, the legacy root is only checked once the borg2 cache root is -confirmed absent for this id, so an unreadable borg2 cache entry reports -\(dqunknown\(dq rather than falling back to check whether a legacy cache exists), or -\(dqunknown\(dq -(this command could not determine which of the above is true \- most commonly a -permission error on the repository\-id directory itself, or on one of the files -inside it). A file that is simply missing (e.g. a repository with only a -\(dqlocation\(dq file and no \(dqkey\-type\(dq) does not affect the other field\(aqs state (that -repository still has \(dqSecurity info: yes\(dq); a security file that exists but -could not be fully read (a permission error, a file too large, or not valid -UTF\-8) does not change \(dqSecurity info\(dq away from \(dqyes\(dq either \- existence, not -readability, is what that field reports. A repository\-id directory that was found -during the directory listing but that then could not be opened or classified for -some other reason \- rather than being dropped from the results, as the directory -listing already proves it exists \- is still listed, with \(dqunknown\(dq for whichever -of \(dqSecurity info\(dq/\(dqLocal cache\(dq/\(dqLayout\(dq could not be established, so a script -auditing this output never mistakes \(dqcould not check\(dq for \(dqconfirmed not there\(dq. -For the cache specifically, once a cache directory\(aqs own existence is confirmed -(a real, non\-symlinked directory containing a \fBconfig\fP file), a later failure -to fully list its \fIcontents\fP does not change \(dqLocal cache\(dq away from \(dqyes\(dq \- -only the reported size becomes a partial count rather than the exact size. Any -of these inspection failures is both logged as its own warning and counted -towards a final summary warning, which (unlike a plain log message) makes this -command exit with a non\-zero return code (usually 1) \- including when the -failure happened while listing a security or cache root itself (e.g. EIO), which -can keep a repository that exists only there out of the output entirely, with no -per\-repository record to point to. A script checking the exit code therefore -learns only that the scan was incomplete, not that every affected repository is -still represented somewhere in the output. -.SH OPTIONS -.sp -See \fIborg\-common(1)\fP for common options of Borg commands. -.SS options -.INDENT 0.0 -.TP -.B \-\-json -format output as JSON -.UNINDENT -.SH EXAMPLES -.INDENT 0.0 -.INDENT 3.5 -.sp -.EX -$ borg known\-repos -Repository ID: 0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 - Layout: borg2 - Location: /path/to/repo - Local cache: yes, 1.23 MB - Cache last written: 2026\-10\-07T21:00:00.000000+09:00 - Security info: yes (aes256\-ocb, sha256) - -$ borg known\-repos \-\-json -{ - \(dqrepositories\(dq: [ - { - \(dqcache_config_mtime\(dq: \(dq2026\-10\-07T21:00:00.000000+09:00\(dq, - \(dqcache_size\(dq: 1234567, - \(dqhas_cache\(dq: true, - \(dqhas_security_info\(dq: true, - \(dqid\(dq: \(dq0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903\(dq, - \(dqkey_type\(dq: \(dqaes256\-ocb, sha256\(dq, - \(dqkey_type_raw\(dq: \(dq16\(dq, - \(dqlayout\(dq: \(dqborg2\(dq, - \(dqlocation\(dq: \(dq/path/to/repo\(dq - } - ] -} -.EE -.UNINDENT -.UNINDENT -.SH SEE ALSO -.sp -\fIborg\-common(1)\fP -.SH Author -The Borg Collective -.\" End of generated man page. diff --git a/docs/usage/known-repos.rst b/docs/usage/known-repos.rst index 70320670a3..28558ddb08 100644 --- a/docs/usage/known-repos.rst +++ b/docs/usage/known-repos.rst @@ -5,26 +5,23 @@ Examples :: $ borg known-repos - Repository ID: 0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 - Layout: borg2 - Location: /path/to/repo - Local cache: yes, 1.23 MB - Cache last written: 2026-10-07T21:00:00.000000+09:00 - Security info: yes (aes256-ocb, sha256) + Repository ID Layout Cache Security Key files Last written Location + 401606173d54eab.. borg2 180 B aes256-ocb, sha256 - 2026-10-12T17:15:19 /path/to/repo $ borg known-repos --json { "repositories": [ { - "cache_config_mtime": "2026-10-07T21:00:00.000000+09:00", - "cache_size": 1234567, + "cache_config_mtime": "2026-10-12T17:15:19.785785+09:00", + "cache_size": 180, "has_cache": true, "has_security_info": true, - "id": "0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903", + "id": "401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb", + "key_file_count": 0, "key_type": "aes256-ocb, sha256", - "key_type_raw": "16", "layout": "borg2", - "location": "/path/to/repo" + "location": "/path/to/repo", + "location_mismatch": false } ] } diff --git a/docs/usage/known-repos.rst.inc b/docs/usage/known-repos.rst.inc deleted file mode 100644 index 67dbf4eacc..0000000000 --- a/docs/usage/known-repos.rst.inc +++ /dev/null @@ -1,132 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_known-repos: - -borg known-repos ----------------- -.. code-block:: none - - borg [common options] known-repos [options] - -.. only:: html - - .. class:: borg-options-table - - +-------------------------------------------------------+------------+-----------------------+ - | **options** | - +-------------------------------------------------------+------------+-----------------------+ - | | ``--json`` | format output as JSON | - +-------------------------------------------------------+------------+-----------------------+ - | .. class:: borg-common-opt-ref | - | | - | :ref:`common_options` | - +-------------------------------------------------------+------------+-----------------------+ - - .. raw:: html - - - -.. only:: latex - - - - options - --json format output as JSON - - - :ref:`common_options` - | - -Description -~~~~~~~~~~~ - -This command lists the repositories that this local Borg installation currently -has state for: security information (key type, last known location) and/or a -local repository cache. - -It inspects only local state - the security directory and the cache directory, -each checked both at its current (borg2) default location and, separately, at -the legacy (borg 1.x) default location those same kinds of state used before the -borg2 base-dir rewrite (see ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` in -``borg help environment`` for the exact resolution rules, including the platforms -where the two locations differ by default, e.g. macOS). It does not access any -repository, locally or remotely, so a repository that is currently unreachable -(e.g. a disconnected external disk or an unresponsive remote host) will still be -listed just fine. This also means -r/--repo has no effect on this command: it -always lists every locally known repository. - -"Layout" reflects which state directory tree the repository was discovered in - -``borg2`` or ``legacy`` - not the repository's actual on-disk format version (see -``borg repo-info``'s "Repository version" for that; this command never opens the -repository, so it cannot report that field at all). The borg2 security entry is -checked first, then the legacy (borg 1.x) security entry, then - only when -neither security entry could be confirmed - whichever cache root matched (borg2 -cache checked before the legacy cache). "unknown" means neither a security entry -nor a cache entry could be confirmed for this id - it was listed as a directory -name, but this command could not establish which layout (if either) it belongs -to. If ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` make the borg2 and legacy -locations resolve to the same path, only the borg2 scan runs and a legacy (v1) -repository there is reported with layout ``borg2``. - -This is useful for auditing which repositories this installation has local state for -(and, for repository-encrypted modes, which crypto suite each one uses), finding -repositories with a local cache that may be taking up unwanted disk space, and general -housekeeping of stale local Borg state. - -The reported cache size is the size of all regular files below the repository's -cache directory (symlinks are not followed and not counted) - for a current (borg2) -repository that is the files cache plus a small amount of metadata (``config``, -``README``), not an index of the repository's chunks; a legacy (borg 1.x) cache -directory can be considerably larger, since it also holds a local chunks index. -A repository-id directory that is a symlink (Borg itself always creates a plain -directory) is skipped, and a ``location`` or ``key-type`` file that is a symlink -is treated the same as missing; neither is followed. - -"Cache last written" is the modification time of the cache's own config file, as -a best-effort proxy for when a cache-using Borg command last ran against this -repository. It is not a precise "last backup" or "last access" time: not every -Borg command updates the cache, and a repository without a local cache at all -will not show one. - -This command does not list or export any key material: the keys dir -(``$BORG_KEYS_DIR``) is not scanned. - -"Security info" and "Local cache" are each one of three states, not a plain -yes/no: "yes" (confirmed present - for security info, a ``location`` and/or -``key-type`` file was confirmed to exist there, independent of whether its -*content* could be read; for cache, a real, non-symlinked cache directory with a -``config`` file was confirmed to exist), "no" (confirmed absent - this command -actually searched the relevant location(s) and found nothing there - for the -cache specifically, the legacy root is only checked once the borg2 cache root is -confirmed absent for this id, so an unreadable borg2 cache entry reports -"unknown" rather than falling back to check whether a legacy cache exists), or -"unknown" -(this command could not determine which of the above is true - most commonly a -permission error on the repository-id directory itself, or on one of the files -inside it). A file that is simply missing (e.g. a repository with only a -"location" file and no "key-type") does not affect the other field's state (that -repository still has "Security info: yes"); a security file that exists but -could not be fully read (a permission error, a file too large, or not valid -UTF-8) does not change "Security info" away from "yes" either - existence, not -readability, is what that field reports. A repository-id directory that was found -during the directory listing but that then could not be opened or classified for -some other reason - rather than being dropped from the results, as the directory -listing already proves it exists - is still listed, with "unknown" for whichever -of "Security info"/"Local cache"/"Layout" could not be established, so a script -auditing this output never mistakes "could not check" for "confirmed not there". -For the cache specifically, once a cache directory's own existence is confirmed -(a real, non-symlinked directory containing a ``config`` file), a later failure -to fully list its *contents* does not change "Local cache" away from "yes" - -only the reported size becomes a partial count rather than the exact size. Any -of these inspection failures is both logged as its own warning and counted -towards a final summary warning, which (unlike a plain log message) makes this -command exit with a non-zero return code (usually 1) - including when the -failure happened while listing a security or cache root itself (e.g. EIO), which -can keep a repository that exists only there out of the output entirely, with no -per-repository record to point to. A script checking the exit code therefore -learns only that the scan was incomplete, not that every affected repository is -still represented somewhere in the output. \ No newline at end of file diff --git a/src/borg/archiver/known_repos_cmd.py b/src/borg/archiver/known_repos_cmd.py index 9d948d4c02..dde2c37695 100644 --- a/src/borg/archiver/known_repos_cmd.py +++ b/src/borg/archiver/known_repos_cmd.py @@ -1,40 +1,29 @@ -import errno import os import re import stat -from contextlib import ExitStack +from configparser import ConfigParser, Error as ConfigParserError from pathlib import Path -from ..constants import * # NOQA -from ..crypto.key import LEGACY_KEY_TYPES, AVAILABLE_KEY_TYPES -from ..helpers import format_file_size, json_print, get_security_dir, get_cache_dir +from ..crypto.key import LEGACY_KEY_TYPES, AVAILABLE_KEY_TYPES, KEYFILE_ID +from ..helpers import format_file_size, json_print, get_security_dir, get_cache_dir, get_keys_dir from ..helpers import safe_timestamp, OutputTimestamp -from ..helpers import os_open, os_stat, flags_dir, flags_dir_follow, flags_normal from ..helpers.argparsing import ArgumentParser -from ..legacy.fs import get_config_dir as get_config_dir_legacy from ..legacy.fs import get_cache_dir as get_cache_dir_legacy +from ..legacy.fs import get_security_dir as get_security_dir_legacy +from ..legacy.fs import get_keys_dir as get_keys_dir_legacy from ..logger import create_logger logger = create_logger() -# A borg2 repository id is a 32 byte random id, hex-encoded (bin_to_hex), see Repository.create(). -# \Z (not $) anchors strictly at the end of the string: $ also matches just before a trailing -# newline, which would let a directory name like "<64 hex chars>\n" slip through re.match(). REPO_ID_RE = re.compile(r"^[0-9a-f]{64}\Z") def _printable(s): - """Escape control characters for safe terminal output (JSON output is unaffected).""" return "".join(c if c.isprintable() else repr(c)[1:-1] for c in s) def _build_key_type_labels(): - """Map each key-type byte (as SecurityManager.save() writes it) to a readable label. - - Derived from LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES (the classes identify_key() searches), - matching how "borg repo-info" builds its "Encrypted:" line, rather than a hand-maintained table. - """ labels = {} for key_cls in LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES: is_legacy = key_cls in LEGACY_KEY_TYPES @@ -48,577 +37,270 @@ def _build_key_type_labels(): return labels -# built once at import time: the key classes list is static. KEY_TYPE_LABELS = _build_key_type_labels() -class _Inspector: - """Filesystem probes that count a failed probe as "incomplete" (see print_warning below). - - Built on borg's os_open()/os_stat() helpers (helpers/fs.py), the same ones create_cmd.py - uses. Where those return a real fd (not Windows), a validated directory is read from via - that fd rather than its pathname - closing the TOCTOU window where the pathname is replaced - between a check and a later read. On Windows, os_open() returns None for directories and a - pathname fallback is used instead - still correct, just not race-free. - """ - +class _Scanner: def __init__(self): - self.incomplete = 0 + self.incomplete = False - def _warn(self, msg, *args): - self.incomplete += 1 - # escape control chars in str/OSError args before logging (logger.py has no sanitizer) + def warn(self, msg, *args): + self.incomplete = True logger.warning(msg, *(_printable(str(a)) if isinstance(a, (str, OSError)) else a for a in args)) - def close_dir(self, d): - """Close a directory handle from open_root/open_subdir - a no-op for the Windows - - pathname fallback (a Path, not a real fd). + def _list_dir(self, root): + """List root's direct children, or [] if it does not exist - an OSError beyond that + (e.g. a configured root that is not actually a directory, EACCES, EIO) is warned on + and also counted via warn(). """ - if not isinstance(d, Path): - os.close(d) + try: + return list(root.iterdir()) + except FileNotFoundError: + return [] + except OSError as exc: + self.warn("known-repos: could not list %s, results may be incomplete: %s", root, exc) + return [] - def _open_dir(self, parent, name, display, follow_symlinks): - """Open name (an absolute Path if parent is None, else relative to the fd/Path parent) + def repo_id_dirs(self, root): + for entry in self._list_dir(root): + if REPO_ID_RE.match(entry.name) and self._is_dir(entry): + yield entry.name, entry - as a directory, via borg's os_open(). None if missing, not a directory, or (if not - follow_symlinks) a symlink. parent is None only for a scan root. The caller must - close_dir() a non-None result. + def _stat(self, path): + """Symlink-following stat(), or None if the path does not exist - an OSError beyond + that (e.g. EACCES, EIO) is warned on and also counted via warn(). """ - flags = flags_dir_follow if follow_symlinks else flags_dir try: - if parent is None: - fd = os_open(path=name, flags=flags) - elif isinstance(parent, Path): - fd = os_open(path=parent / name, flags=flags) # Windows pathname fallback - else: - fd = os_open(parent_fd=parent, name=name, flags=flags) - except (FileNotFoundError, NotADirectoryError): - return None - except OSError as exc: - if not follow_symlinks and getattr(exc, "errno", None) == errno.ELOOP: - return None # O_NOFOLLOW rejected a symlink leaf - not a failure, just "not a dir" - self._warn("known-repos: could not open %s: %s", display, exc) + return path.stat() + except FileNotFoundError: return None - if fd is None: - # Windows: os_open() can't open a directory; fall back to the pathname itself. - path = name if parent is None else (parent / name if isinstance(parent, Path) else None) - if path is None: - self._warn("known-repos: could not open %s: no descriptor and no pathname fallback", display) - return None - try: - st = path.stat() if follow_symlinks else path.lstat() - except (FileNotFoundError, NotADirectoryError): - return None - except OSError as exc: - self._warn("known-repos: could not check %s: %s", display, exc) - return None - return path if stat.S_ISDIR(st.st_mode) else None - # release fd on every path out except the final success, including KeyboardInterrupt etc. - try: - is_real_dir = stat.S_ISDIR(os.fstat(fd).st_mode) except OSError as exc: - os.close(fd) - self._warn("known-repos: could not check %s: %s", display, exc) - return None - except BaseException: - os.close(fd) - raise - if not is_real_dir: - os.close(fd) + self.warn("known-repos: could not check %s: %s", path, exc) return None - return fd - def open_root(self, path, display): - """Open a scan root (security/cache root) by absolute path, following symlinks - these - - are Borg's own config/cache directories, not attacker-influenced repo-id entries, so - there is nothing to protect by rejecting a symlink here. None if not a real directory. - """ - return self._open_dir(None, path, display, follow_symlinks=True) + def _is_dir(self, path): + st = self._stat(path) + return st is not None and stat.S_ISDIR(st.st_mode) - def open_subdir(self, parent, name, display): - """Open and validate name below parent as a real (non-symlink) directory. None if + def is_real_file(self, path): + st = self._stat(path) + return st is not None and stat.S_ISREG(st.st_mode) - missing/not-a-real-directory. The caller must close_dir() a non-None result. + def marker_exists(self, path): + """True if the marker is confirmed present, False otherwise - including on a + filesystem error, which is already warned on and counted via self._stat(). """ - return self._open_dir(parent, name, display, follow_symlinks=False) + return self._stat(path) is not None - def read_subfile(self, parent, name, display, max_size=4096): - """Read name below parent as a real (non-symlink) regular file, bounded. - - None if missing, not a regular file, too large, or not UTF-8. Missing and - "not a regular file" (including a symlink/FIFO in place of the expected file, - since Borg itself never creates one there) are not inspection failures; too - large and not-UTF-8 are, and so is a read/stat error on an entry that does - exist. - """ - if isinstance(parent, Path): - path = parent / name - try: - st = path.lstat() - except (FileNotFoundError, NotADirectoryError): - return None - except OSError as exc: - self._warn("known-repos: could not check %s: %s", display, exc) - return None - if not stat.S_ISREG(st.st_mode): - return None - try: - with path.open("rb") as f: - data = f.read(max_size + 1) - except (FileNotFoundError, NotADirectoryError): - return None - except OSError as exc: - self._warn("known-repos: could not read %s: %s", display, exc) - return None - else: - # flags_normal includes O_NONBLOCK|O_NOFOLLOW (see helpers/fs.py) - the same flags - # borg's own create_cmd.py uses to read a regular file without blocking on a FIFO - # that has no writer, and without following a symlink leaf. - try: - fd = os_open(parent_fd=parent, name=name, flags=flags_normal) - except (FileNotFoundError, NotADirectoryError): - return None - except OSError as exc: - if getattr(exc, "errno", None) == errno.ELOOP: - return None - self._warn("known-repos: could not read %s: %s", display, exc) - return None - try: - if not stat.S_ISREG(os.fstat(fd).st_mode): - return None - # loop until EOF or max_size + 1 bytes: a single os.read can return a short read - chunks = [] - remaining = max_size + 1 - while remaining > 0: - chunk = os.read(fd, remaining) - if not chunk: - break - chunks.append(chunk) - remaining -= len(chunk) - data = b"".join(chunks) - except OSError as exc: - self._warn("known-repos: could not read %s: %s", display, exc) - return None - finally: - os.close(fd) - if len(data) > max_size: - self._warn("known-repos: %s is larger than %d bytes, ignoring its content", display, max_size) - return None + def read_text(self, path): try: - return data.decode("utf-8") + return path.read_text(encoding="utf-8") + except FileNotFoundError: + return None + except OSError as exc: + self.warn("known-repos: could not read %s: %s", path, exc) + return None except UnicodeDecodeError as exc: - self._warn("known-repos: %s is not valid UTF-8, ignoring its content: %s", display, exc) + self.warn("known-repos: %s is not valid UTF-8, ignoring its content: %s", path, exc) return None - def stat_subfile(self, parent, name, display): - """stat (not following symlinks) name below parent; the result only if it is a regular file.""" - if isinstance(parent, Path): - path = parent / name - try: - st = path.lstat() - except (FileNotFoundError, NotADirectoryError): - return None - except OSError as exc: - self._warn("known-repos: could not check %s: %s", display, exc) - return None - else: - try: - st = os_stat(parent_fd=parent, name=name, follow_symlinks=False) - except (FileNotFoundError, NotADirectoryError): - return None - except OSError as exc: - self._warn("known-repos: could not check %s: %s", display, exc) - return None - return st if stat.S_ISREG(st.st_mode) else None - - def list_names(self, d, display): - """List entry names in an open directory (fd or Path), or [] with a warning on OSError.""" + def read_first_line(self, path, max_size=128): + """Read only the first max_size bytes of a (possibly large) file and return the line + up to the first newline within them - used for key files, so most of a key file's + content (the encrypted key material after the header) is never read. The header + itself ("BORG_KEY ", 73 bytes) always fits within max_size, so this never + needs to read past it in practice. + """ try: - if isinstance(d, Path): - return [e.name for e in d.iterdir()] - with os.scandir(d) as it: - return [e.name for e in it] - except (FileNotFoundError, NotADirectoryError): - return [] + with open(path, "rb") as fd: + chunk = fd.read(max_size) + except FileNotFoundError: + return None except OSError as exc: - self._warn("known-repos: could not list %s, results may be incomplete: %s", display, exc) - return [] + self.warn("known-repos: could not read %s: %s", path, exc) + return None + line = chunk.split(b"\n", 1)[0] + try: + return line.decode("utf-8") + except UnicodeDecodeError: + return None - def dir_size(self, root, root_display): - """Sum regular-file sizes below an already-open/validated directory (fd or Path), not + def plain_files(self, root): + """Yield (name, path) for every regular file directly under root.""" + for entry in self._list_dir(root): + if self.is_real_file(entry): + yield entry.name, entry - following symlinks. Consumes (closes) root. Best-effort: an unreadable entry is - skipped (warned, counted incomplete), so the total can be a partial count. Uses an - explicit stack, not Python recursion, to bound open descriptors by tree depth, not - breadth, and to avoid Python's recursion limit on a very deep tree. - """ + def mtime(self, path): + st = self._stat(path) + return OutputTimestamp(safe_timestamp(st.st_mtime_ns)) if st is not None else None + + def dir_size(self, path): total = 0 - # each entry is (dir_handle, display, remaining_names_iterator); closed once exhausted. - # root is pushed before the try so a BaseException from list_names() (list_names() itself - # catches OSError) still leaves root on the stack for the finally below to close. - stack = [[root, root_display, None]] - try: - stack[0][2] = iter(self.list_names(root, root_display)) - while stack: - current, display, names = stack[-1] - name = next(names, None) - if name is None: - self.close_dir(current) - stack.pop() - continue - entry_display = f"{display}/{name}" - if isinstance(current, Path): - try: - st = (current / name).lstat() - except (FileNotFoundError, NotADirectoryError): - continue - except OSError as exc: - self._warn( - "known-repos: could not check %s, reported cache size may be incomplete: %s", - entry_display, - exc, - ) - continue - else: - try: - st = os_stat(parent_fd=current, name=name, follow_symlinks=False) - except (FileNotFoundError, NotADirectoryError): - continue - except OSError as exc: - self._warn( - "known-repos: could not check %s, reported cache size may be incomplete: %s", - entry_display, - exc, - ) - continue - if stat.S_ISDIR(st.st_mode): - sub = self.open_subdir(current, name, entry_display) - if sub is not None: - frame = [sub, entry_display, None] - stack.append(frame) - frame[2] = iter(self.list_names(sub, entry_display)) - elif stat.S_ISREG(st.st_mode): - total += st.st_size - finally: - # closes anything left on the stack on an exception path (ordinary completion - # already pops/closes each level). - for entry in stack: - self.close_dir(entry[0]) + for dirpath, dirnames, filenames in os.walk(path, onerror=self._walk_error): + for name in filenames: + fpath = os.path.join(dirpath, name) + try: + total += os.path.getsize(fpath) + except OSError as exc: + self.warn("known-repos: could not check %s, reported cache size may be incomplete: %s", fpath, exc) return total - def cache_config_mtime(self, cache_dir, display): - """mtime of config below an already-open cache_dir, as an OutputTimestamp, or None. + def _walk_error(self, exc): + self.warn("known-repos: could not list %s, reported cache size may be incomplete: %s", exc.filename, exc) - Best-effort proxy for "a cache-using borg command last ran" - not every command touches - the cache, so this can lag behind actual repository access. - """ - st = self.stat_subfile(cache_dir, "config", f"{display}/config") - if st is None: - return None - try: - return OutputTimestamp(safe_timestamp(st.st_mtime_ns)) - except (OverflowError, OSError, ValueError) as exc: - self._warn("known-repos: could not convert the mtime of %s/config: %s", display, exc) - return None - def open_cache_entry(self, cache_root, repo_id, display): - """Open and classify the cache entry for repo_id below an already-open cache_root. +def _scan_security_root(scanner, root): + info = {} + for repo_id, entry in scanner.repo_id_dirs(root): + location_path = entry / "location" + key_type_path = entry / "key-type" + has_location = scanner.is_real_file(location_path) + has_key_type = scanner.is_real_file(key_type_path) + if not (has_location or has_key_type): + continue + location = scanner.read_text(location_path) if has_location else None + key_type_raw = scanner.read_text(key_type_path) if has_key_type else None + info[repo_id] = (location or None, (key_type_raw.strip() or None) if key_type_raw else None) + return info - Returns (kind, entry): 'present' (entry is the open, caller-owned handle for - dir_size/cache_config_mtime), 'absent' (confirmed not a cache, entry is None), or - 'unreadable' (state never established - not a confirmed cache, and the caller must not - fall back to another root either; entry is None). - """ - before = self.incomplete - entry = self.open_subdir(cache_root, repo_id, display) - if entry is None: - return ("unreadable" if self.incomplete > before else "absent"), None - # release entry on every path out except the "present" return, which transfers ownership - try: - before = self.incomplete - if self.stat_subfile(entry, "config", f"{display}/config") is not None: - return "present", entry - kind = "unreadable" if self.incomplete > before else "absent" - except BaseException: - self.close_dir(entry) - raise - self.close_dir(entry) - return kind, None - - def scan_security_root(self, root, root_display): - """Return (repo_ids, info, unreadable_ids, listing_ok) for ids directly below an - already-open security root. - - repo_ids is the set of 64-hex-digit directories with at least one of the two known files - confirmed present (checked via stat_subfile, independent of read success). info is - {repo_id: (location, key_type_raw)}, read while the entry is open; a value is None if - its file is absent or could not be read/decoded. - - unreadable_ids is disjoint from repo_ids: directory names list_names() returned whose - presence/absence could not be established (the entry itself could not be opened, e.g. - EACCES, or neither file was confirmed present and at least one of the two - presence checks failed). A name list_names() returned must end up in repo_ids, - unreadable_ids, or genuinely neither - never silently dropped. - - listing_ok is False when list_names() on root itself failed (e.g. EIO after the root was - already opened): repo_ids/unreadable_ids are then necessarily incomplete, so the caller - must not treat this root as fully searched for an id discovered elsewhere. - """ - repo_ids = set() - info = {} - unreadable_ids = set() - before_list = self.incomplete - names = self.list_names(root, root_display) - listing_ok = self.incomplete == before_list - for name in names: - if not REPO_ID_RE.match(name): - continue - entry_display = f"{root_display}/{name}" - before = self.incomplete - entry = self.open_subdir(root, name, entry_display) - if entry is None: - if self.incomplete > before: - unreadable_ids.add(name) - continue - try: - before = self.incomplete - has_location = self.stat_subfile(entry, "location", f"{entry_display}/location") is not None - has_key_type = self.stat_subfile(entry, "key-type", f"{entry_display}/key-type") is not None - if has_location or has_key_type: - location = self.read_subfile(entry, "location", f"{entry_display}/location") - # SecurityManager.save() writes location verbatim (no trailing newline); only - # turn a present-but-empty read into None, do not alter legitimate whitespace. - location = location or None - key_type_raw = self.read_subfile(entry, "key-type", f"{entry_display}/key-type") - key_type_raw = key_type_raw.strip() or None if key_type_raw else None - repo_ids.add(name) - info[name] = (location, key_type_raw) - elif self.incomplete > before: - # stat_subfile only counts incomplete on a real OSError, not on - # FileNotFoundError/NotADirectoryError, so this path means a genuine - # EACCES-class failure, not plain absence. - unreadable_ids.add(name) - # else: neither file exists - not a security entry, not an inspection failure. - finally: - self.close_dir(entry) - return repo_ids, info, unreadable_ids, listing_ok - - def scan_cache_root(self, root, root_display, repo_ids_out, unreadable_ids_out): - """Add repo ids directly below an already-open cache root into repo_ids_out: a - - 64-hex-digit directory with a config *file* (excludes the storecache directory and - the CACHEDIR.TAG file get_cache_dir() writes). A name list_names() returned that could - not be classified as having a cache or not (the entry could not be opened, or its - config file's presence could not be established) goes into unreadable_ids_out instead - of being dropped - see scan_security_root() for why that distinction matters. - - Returns listing_ok (see scan_security_root() for its meaning). - """ - before_list = self.incomplete - names = self.list_names(root, root_display) - listing_ok = self.incomplete == before_list - for name in names: - if not REPO_ID_RE.match(name): - continue - entry_display = f"{root_display}/{name}" - before = self.incomplete - entry = self.open_subdir(root, name, entry_display) - if entry is None: - if self.incomplete > before: - unreadable_ids_out.add(name) - continue - try: - before = self.incomplete - is_cache = self.stat_subfile(entry, "config", f"{entry_display}/config") is not None - finally: - self.close_dir(entry) - if is_cache: - repo_ids_out.add(name) - elif self.incomplete > before: - unreadable_ids_out.add(name) - return listing_ok - - -def _legacy_security_dir(): - """Where borg 1.x stores local security information, without creating anything. - - Mirrors legacy.fs.get_security_dir(create=False), but that still calls get_config_dir() - with create=True internally; re-implemented here to stay fully read-only. + +def _scan_cache_root(scanner, root): + return {repo_id: entry for repo_id, entry in scanner.repo_id_dirs(root) if scanner.is_real_file(entry / "config")} + + +def _scan_keys_dir(scanner, root): + """Map repo_id -> set of key file paths, by reading only the first 128 bytes of each file + (enough to cover the "BORG_KEY " header, never the bulk of the encrypted key + material that follows it). A key file's name is a hash of its own content, not the repo + id, so this has to open every file in the directory - unlike the security/cache dirs, + which name their subdirectories after the repo id. + """ + info = {} + for _name, path in scanner.plain_files(root): + header = scanner.read_first_line(path) + if not header or not header.startswith(f"{KEYFILE_ID} "): + continue + repo_id = header[len(KEYFILE_ID) + 1 :].strip() + if not REPO_ID_RE.match(repo_id): + continue + info.setdefault(repo_id, set()).add(path) + return info + + +def _cache_layout(scanner, cache_entry): + """ "legacy" if the cache entry has a borg 1.x marker ("chunks.archive.d" or a "chunks" + file), else "borg2". A marker that could not be confirmed due to a filesystem error is + treated the same as absent (the error is already warned on by marker_exists()). """ - security_dir = os.environ.get("BORG_SECURITY_DIR") - if security_dir is None: - security_dir = str(Path(get_config_dir_legacy(create=False)) / "security") - return security_dir + if scanner.marker_exists(cache_entry / "chunks.archive.d"): + return "legacy" + return "legacy" if scanner.marker_exists(cache_entry / "chunks") else "borg2" + + +def _cache_previous_location(scanner, cache_entry): + config_text = scanner.read_text(cache_entry / "config") + if not config_text: + return None + parser = ConfigParser(interpolation=None) + try: + parser.read_string(config_text) + return parser.get("cache", "previous_location", fallback=None) or None + except ConfigParserError as exc: + scanner.warn("known-repos: could not parse %s: %s", cache_entry / "config", exc) + return None class KnownReposMixIn: def do_known_repos(self, args): """List repositories Borg has local state for (security info and/or cache).""" - inspector = _Inspector() + scanner = _Scanner() + security_root = Path(get_security_dir(create=False)) - # v1 repos keep security info at a different default location than borg2 (and on some - # platforms, e.g. macOS, a different cache root too) unless BORG_SECURITY_DIR/BORG_CACHE_DIR - # override both to the same place. - legacy_security_root = Path(_legacy_security_dir()) + legacy_security_root = Path(get_security_dir_legacy(create=False)) cache_root = Path(get_cache_dir(create=False)) legacy_cache_root = Path(get_cache_dir_legacy(create=False)) - - with ExitStack() as stack: - # every root is opened once up front and held open for the whole scan. - def _open(path): - before = inspector.incomplete - d = inspector.open_root(path, str(path)) - if d is not None: - stack.callback(inspector.close_dir, d) - return d, "present" - return None, ("unreadable" if inspector.incomplete > before else "absent") - - security_root_d, security_root_kind = _open(security_root) - if legacy_security_root != security_root: - legacy_security_root_d, legacy_security_root_kind = _open(legacy_security_root) + keys_root = Path(get_keys_dir(create=False)) + legacy_keys_root = Path(get_keys_dir_legacy(create=False)) + + security_roots_coincide = legacy_security_root == security_root + cache_roots_coincide = legacy_cache_root == cache_root + keys_roots_coincide = legacy_keys_root == keys_root + + borg2_security = _scan_security_root(scanner, security_root) + legacy_security = {} if security_roots_coincide else _scan_security_root(scanner, legacy_security_root) + + borg2_cache = _scan_cache_root(scanner, cache_root) + legacy_cache = {} if cache_roots_coincide else _scan_cache_root(scanner, legacy_cache_root) + legacy_cache = {k: v for k, v in legacy_cache.items() if k not in borg2_cache} + + keys = _scan_keys_dir(scanner, keys_root) + if not keys_roots_coincide: + for repo_id, paths in _scan_keys_dir(scanner, legacy_keys_root).items(): + keys.setdefault(repo_id, set()).update(paths) + + repo_ids = set(borg2_security) | set(legacy_security) | set(borg2_cache) | set(legacy_cache) | set(keys) + + repos = [] + for repo_id in sorted(repo_ids): + if repo_id in borg2_security: + location, key_type_raw = borg2_security[repo_id] + has_security_info = True + # location/key-type filenames are the same for borg1 and borg2, so a shared + # root cannot tell them apart - defer to the cache-based check below. + layout = None if security_roots_coincide else "borg2" + elif repo_id in legacy_security: + location, key_type_raw = legacy_security[repo_id] + layout = "legacy" + has_security_info = True else: - legacy_security_root_d, legacy_security_root_kind = None, "absent" - cache_root_d, cache_root_kind = _open(cache_root) - if legacy_cache_root != cache_root: - legacy_cache_root_d, legacy_cache_root_kind = _open(legacy_cache_root) - else: - legacy_cache_root_d, legacy_cache_root_kind = None, "absent" - - if security_root_d is not None: - borg2_ids, borg2_info, borg2_unreadable_ids, borg2_security_listing_ok = inspector.scan_security_root( - security_root_d, str(security_root) - ) - else: - borg2_ids, borg2_info, borg2_unreadable_ids, borg2_security_listing_ok = set(), {}, set(), True - if legacy_security_root_d is not None: - legacy_ids, legacy_info, legacy_unreadable_ids, legacy_security_listing_ok = ( - inspector.scan_security_root(legacy_security_root_d, str(legacy_security_root)) - ) - legacy_ids -= borg2_ids # prefer the borg2 entry if an id somehow exists in both trees - legacy_unreadable_ids -= borg2_ids - else: - legacy_ids, legacy_info, legacy_unreadable_ids, legacy_security_listing_ok = set(), {}, set(), True - security_unreadable_ids = (borg2_unreadable_ids | legacy_unreadable_ids) - borg2_ids - legacy_ids - repo_ids = borg2_ids | legacy_ids | security_unreadable_ids - # "no security entry" can only be a confirmed False for a cache-only id when BOTH - # security roots were fully searched (root opened and its own listing succeeded, - # not just free of per-id EACCES - scan_security_root() tracks that separately). - security_fully_searched = ( - security_root_kind != "unreadable" - and legacy_security_root_kind != "unreadable" - and borg2_security_listing_ok - and legacy_security_listing_ok - ) - borg2_cache_ids = set() - borg2_cache_unreadable_ids = set() - if cache_root_d is not None: - inspector.scan_cache_root(cache_root_d, str(cache_root), borg2_cache_ids, borg2_cache_unreadable_ids) - legacy_cache_ids = set() - legacy_cache_unreadable_ids = set() - if legacy_cache_root_d is not None: - inspector.scan_cache_root( - legacy_cache_root_d, str(legacy_cache_root), legacy_cache_ids, legacy_cache_unreadable_ids - ) - cache_unreadable_ids = (borg2_cache_unreadable_ids | legacy_cache_unreadable_ids) - ( - borg2_cache_ids | legacy_cache_ids - ) - repo_ids |= borg2_cache_ids | legacy_cache_ids | cache_unreadable_ids - - repos = [] - for repo_id in sorted(repo_ids): - if repo_id in borg2_ids: - location, key_type_raw = borg2_info[repo_id] - layout = "borg2" - has_security_info = True - elif repo_id in legacy_ids: - location, key_type_raw = legacy_info[repo_id] - layout = "legacy" - has_security_info = True - elif repo_id in borg2_cache_ids: - location, key_type_raw = None, None - layout = "borg2" - # discovered only via the cache; security info is confirmed absent only if - # this id was not flagged unreadable AND both security roots were fully - # searched (see security_fully_searched above). - if repo_id in security_unreadable_ids or not security_fully_searched: - has_security_info = None - else: - has_security_info = False - elif repo_id in legacy_cache_ids: - location, key_type_raw = None, None - layout = "legacy" - if repo_id in security_unreadable_ids or not security_fully_searched: - has_security_info = None + location, key_type_raw = None, None + layout = None + has_security_info = False + + cache_entry = borg2_cache.get(repo_id) or legacy_cache.get(repo_id) + has_cache = cache_entry is not None + cache_size = None + cache_mtime = None + location_mismatch = False + if has_cache: + cache_size = scanner.dir_size(cache_entry) + cache_mtime = scanner.mtime(cache_entry / "config") + if layout is None: + if cache_roots_coincide: + layout = _cache_layout(scanner, cache_entry) else: - has_security_info = False - else: - location, key_type_raw = None, None - layout = None - if repo_id in security_unreadable_ids or not security_fully_searched: - has_security_info = None + layout = "legacy" if repo_id in legacy_cache else "borg2" + if layout == "legacy": + cache_location = _cache_previous_location(scanner, cache_entry) + if location is None: + location = cache_location else: - has_security_info = False - - # fall back to the legacy cache root only when borg2 is confirmed absent - - # "unreadable" must not be treated as "try the other root" nor as "present". - used_root_path = cache_root - if cache_root_d is not None: - kind, entry = inspector.open_cache_entry(cache_root_d, repo_id, str(cache_root / repo_id)) - else: - kind, entry = cache_root_kind, None - if kind == "absent" and legacy_cache_root_d is not None: - used_root_path = legacy_cache_root - kind, entry = inspector.open_cache_entry( - legacy_cache_root_d, repo_id, str(legacy_cache_root / repo_id) - ) - elif kind == "absent" and legacy_cache_root_kind == "unreadable": - # current-root confirmed absent, but the legacy root could not be opened - # either - the legacy side was never searched, so this is unknown, not absent. - kind = "unreadable" - - # tri-state: True (confirmed present), False (confirmed absent in every root - # that could be searched), or None (at least one root's state could not be - # established). This re-examines repo_id's cache state fresh rather than - # reusing scan_cache_root()'s earlier discovery pass. - has_cache = {"present": True, "unreadable": None, "absent": False}[kind] - cache_size = None - cache_mtime = None - if has_cache is True: - # dir_size closes entry unconditionally; guard only the window before it - # starts, where cache_config_mtime could raise first. - entry_display = str(used_root_path / repo_id) - try: - cache_mtime = inspector.cache_config_mtime(entry, entry_display) - except BaseException: - inspector.close_dir(entry) - raise - cache_size = inspector.dir_size(entry, entry_display) # consumes/closes entry; always an int - elif entry is not None: - inspector.close_dir(entry) - - repos.append( - { - "id": repo_id, - "layout": layout, - "location": location, - "key_type_raw": key_type_raw, - "key_type": (KEY_TYPE_LABELS.get(key_type_raw) or key_type_raw) if key_type_raw else None, - "has_security_info": has_security_info, - "has_cache": has_cache, - "cache_size": cache_size if has_cache is True else None, - "cache_config_mtime": cache_mtime.isoformat() if has_cache is True and cache_mtime else None, - } - ) + location_mismatch = cache_location is not None and cache_location != location + + repos.append( + { + "id": repo_id, + "layout": layout, + "location": location, + "key_type": ( + (KEY_TYPE_LABELS.get(key_type_raw) or f"unknown key type {key_type_raw}") + if key_type_raw + else None + ), + "has_security_info": has_security_info, + "has_cache": has_cache, + "cache_size": cache_size, + "cache_config_mtime": cache_mtime.isoformat() if cache_mtime else None, + "key_file_count": len(keys.get(repo_id, ())), + "location_mismatch": location_mismatch, + } + ) - if inspector.incomplete: - # counts failed inspection operations, not distinct paths (the same path can be - # probed more than once), so this number can exceed the paths actually affected. + if scanner.incomplete: self.print_warning( - "known-repos: %d filesystem inspection(s) could not be fully completed; " - "results may be incomplete - see the warnings above.", - inspector.incomplete, + "known-repos: some filesystem inspections could not be completed; " + "results may be incomplete - see the warnings above." ) if args.json: @@ -629,125 +311,45 @@ def _open(path): print("No locally known repositories.") return + headers = ("Repository ID", "Layout", "Cache", "Security", "Key files", "Last written") + rows = [] for repo in repos: - print(f"Repository ID: {repo['id']}") - layout = repo["layout"] - if layout is None: - print(" Layout: unknown (could not be determined)") - else: - print(f" Layout: {layout}") - print(f" Location: {_printable(repo['location']) if repo['location'] else '(unknown)'}") - if repo["has_cache"] is True: - print(f" Local cache: yes, {format_file_size(repo['cache_size'])}") - print(f" Cache last written: {repo['cache_config_mtime'] or '(unknown)'}") - elif repo["has_cache"] is None: - print(" Local cache: unknown (could not be determined)") - else: - print(" Local cache: no") - if repo["has_security_info"] is True: - key_type = repo["key_type"] - if key_type: - print(f" Security info: yes ({_printable(key_type)})") - else: - print(" Security info: yes (key type unknown)") - elif repo["has_security_info"] is None: - print(" Security info: unknown (could not be determined)") - else: - print(" Security info: no") - if repo["has_cache"] is True and repo["has_security_info"] is False: - print(" Note: a local cache exists, but no security info was found for this repository.") - print() + security = "no" + if repo["has_security_info"]: + security = _printable(repo["key_type"]) if repo["key_type"] else "yes (key type unknown)" + location = _printable(repo["location"]) if repo["location"] else "(unknown)" + if repo["location_mismatch"]: + location += " (!= cache's last location)" + rows.append( + ( + repo["id"][:15] + "..", + repo["layout"] or "unknown", + format_file_size(repo["cache_size"]) if repo["has_cache"] else "-", + security, + str(repo["key_file_count"]) if repo["key_file_count"] else "-", + repo["cache_config_mtime"][:19] if repo["cache_config_mtime"] else "-", + location, + ) + ) + + widths = [max(len(h), *(len(row[i]) for row in rows)) for i, h in enumerate(headers)] + print("".join(f"{h:<{w}} " for h, w in zip(headers, widths)) + "Location") + for row in rows: + print("".join(f"{col:<{w}} " for col, w in zip(row[:-1], widths)) + row[-1]) def build_parser_known_repos(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog known_repos_epilog = process_epilog(""" - This command lists the repositories that this local Borg installation currently - has state for: security information (key type, last known location) and/or a - local repository cache. - - It inspects only local state - the security directory and the cache directory, - each checked both at its current (borg2) default location and, separately, at - the legacy (borg 1.x) default location those same kinds of state used before the - borg2 base-dir rewrite (see ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` in - ``borg help environment`` for the exact resolution rules, including the platforms - where the two locations differ by default, e.g. macOS). It does not access any - repository, locally or remotely, so a repository that is currently unreachable - (e.g. a disconnected external disk or an unresponsive remote host) will still be - listed just fine. This also means -r/--repo has no effect on this command: it - always lists every locally known repository. - - "Layout" reflects which state directory tree the repository was discovered in - - ``borg2`` or ``legacy`` - not the repository's actual on-disk format version (see - ``borg repo-info``'s "Repository version" for that; this command never opens the - repository, so it cannot report that field at all). The borg2 security entry is - checked first, then the legacy (borg 1.x) security entry, then - only when - neither security entry could be confirmed - whichever cache root matched (borg2 - cache checked before the legacy cache). "unknown" means neither a security entry - nor a cache entry could be confirmed for this id - it was listed as a directory - name, but this command could not establish which layout (if either) it belongs - to. If ``$BORG_SECURITY_DIR``/``$BORG_CACHE_DIR`` make the borg2 and legacy - locations resolve to the same path, only the borg2 scan runs and a legacy (v1) - repository there is reported with layout ``borg2``. - - This is useful for auditing which repositories this installation has local state for - (and, for repository-encrypted modes, which crypto suite each one uses), finding - repositories with a local cache that may be taking up unwanted disk space, and general - housekeeping of stale local Borg state. - - The reported cache size is the size of all regular files below the repository's - cache directory (symlinks are not followed and not counted) - for a current (borg2) - repository that is the files cache plus a small amount of metadata (``config``, - ``README``), not an index of the repository's chunks; a legacy (borg 1.x) cache - directory can be considerably larger, since it also holds a local chunks index. - A repository-id directory that is a symlink (Borg itself always creates a plain - directory) is skipped, and a ``location`` or ``key-type`` file that is a symlink - is treated the same as missing; neither is followed. - - "Cache last written" is the modification time of the cache's own config file, as - a best-effort proxy for when a cache-using Borg command last ran against this - repository. It is not a precise "last backup" or "last access" time: not every - Borg command updates the cache, and a repository without a local cache at all - will not show one. - - This command does not list or export any key material: the keys dir - (``$BORG_KEYS_DIR``) is not scanned. - - "Security info" and "Local cache" are each one of three states, not a plain - yes/no: "yes" (confirmed present - for security info, a ``location`` and/or - ``key-type`` file was confirmed to exist there, independent of whether its - *content* could be read; for cache, a real, non-symlinked cache directory with a - ``config`` file was confirmed to exist), "no" (confirmed absent - this command - actually searched the relevant location(s) and found nothing there - for the - cache specifically, the legacy root is only checked once the borg2 cache root is - confirmed absent for this id, so an unreadable borg2 cache entry reports - "unknown" rather than falling back to check whether a legacy cache exists), or - "unknown" - (this command could not determine which of the above is true - most commonly a - permission error on the repository-id directory itself, or on one of the files - inside it). A file that is simply missing (e.g. a repository with only a - "location" file and no "key-type") does not affect the other field's state (that - repository still has "Security info: yes"); a security file that exists but - could not be fully read (a permission error, a file too large, or not valid - UTF-8) does not change "Security info" away from "yes" either - existence, not - readability, is what that field reports. A repository-id directory that was found - during the directory listing but that then could not be opened or classified for - some other reason - rather than being dropped from the results, as the directory - listing already proves it exists - is still listed, with "unknown" for whichever - of "Security info"/"Local cache"/"Layout" could not be established, so a script - auditing this output never mistakes "could not check" for "confirmed not there". - For the cache specifically, once a cache directory's own existence is confirmed - (a real, non-symlinked directory containing a ``config`` file), a later failure - to fully list its *contents* does not change "Local cache" away from "yes" - - only the reported size becomes a partial count rather than the exact size. Any - of these inspection failures is both logged as its own warning and counted - towards a final summary warning, which (unlike a plain log message) makes this - command exit with a non-zero return code (usually 1) - including when the - failure happened while listing a security or cache root itself (e.g. EIO), which - can keep a repository that exists only there out of the output entirely, with no - per-repository record to point to. A script checking the exit code therefore - learns only that the scan was incomplete, not that every affected repository is - still represented somewhere in the output. + Lists the repositories this local Borg installation currently has state for - + security info (key type, last known location), a local repository cache, and/or key + files - without accessing any repository itself (-r/--repo has no effect). "Layout" + is which state was found in (not the on-disk repository format, see "borg repo-info" + for that); "Last written" is the cache config's mtime, a best-effort last-used proxy, + not an exact timestamp and not a staleness check on its own; a mismatch between a + legacy repo's known location and its cache's last location is flagged; a key file's + presence does not by itself mean the repository is still openable. Use --json for a + machine-readable version, including the full repository id. """) subparser = ArgumentParser( parents=[common_parser], description=self.do_known_repos.__doc__, epilog=known_repos_epilog diff --git a/src/borg/legacy/fs.py b/src/borg/legacy/fs.py index 0939bfc908..0b3693204b 100644 --- a/src/borg/legacy/fs.py +++ b/src/borg/legacy/fs.py @@ -1,9 +1,11 @@ -"""borg 1.x directory layout, kept only so borg transfer can read v1 repos. +"""borg 1.x directory layout, kept only so borg transfer (and now also the +known-repos command) can read v1 repo state. The borg2 directory functions live in helpers.fs. This module keeps the old -home/base, config, cache, keys and security dir resolution. borg.security is -the only caller, and only when repository.version == 1. It goes away with the -rest of the borg 1.x transfer support. +home/base, config, cache, keys and security dir resolution. Callers are +borg.security (only when repository.version == 1) and +borg.archiver.known_repos_cmd (read-only, create=False). It goes away with +the rest of the borg 1.x transfer support. """ import os @@ -126,7 +128,7 @@ def get_keys_dir(*, create=True): keys_dir = os.environ.get("BORG_KEYS_DIR") if keys_dir is None: # note: do not just give this as default to the environment.get(), see issue #5979. - keys_dir = str(Path(get_config_dir()) / "keys") + keys_dir = str(Path(get_config_dir(create=create)) / "keys") if create: ensure_dir(keys_dir) return keys_dir @@ -137,7 +139,7 @@ def get_security_dir(repository_id=None, *, create=True): security_dir = os.environ.get("BORG_SECURITY_DIR") if security_dir is None: # note: do not just give this as default to the environment.get(), see issue #5979. - security_dir = str(Path(get_config_dir()) / "security") + security_dir = str(Path(get_config_dir(create=create)) / "security") if repository_id: security_dir = str(Path(security_dir) / repository_id) if create: diff --git a/src/borg/testsuite/archiver/known_repos_cmd_test.py b/src/borg/testsuite/archiver/known_repos_cmd_test.py index 77f15e754f..27f2a955ba 100644 --- a/src/borg/testsuite/archiver/known_repos_cmd_test.py +++ b/src/borg/testsuite/archiver/known_repos_cmd_test.py @@ -1,28 +1,41 @@ import json import os import shutil - -try: - import resource -except ImportError: # Windows has no resource module - resource = None +from pathlib import Path import pytest -from ...constants import * # NOQA -from ...helpers import get_cache_dir, get_security_dir, format_file_size -from .. import are_symlinks_supported +from ...helpers import get_cache_dir, get_security_dir from . import cmd, create_regular_file, exec_cmd, generate_archiver_tests, RK_ENCRYPTION pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA -requires_symlinks = pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported") -# evaluate geteuid only on POSIX: the name does not exist on Windows, and putting -# os.geteuid() in a skipif condition would raise at collection time there. -requires_chmod_permissions = pytest.mark.skipif( - os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() == 0, - reason="chmod-based permission test, not meaningful on Windows or as root", -) + +def _write_legacy_security(repo_id, location, key_type="3"): + """Create a borg-1.x-style $BORG_CONFIG_DIR/security// entry with the given + location and key-type (default "3" == KeyType.REPO, legacy AES-CTR repokey). + """ + from borg.legacy.fs import get_config_dir as get_config_dir_legacy + + security_dir = os.path.join(get_config_dir_legacy(create=False), "security", repo_id) + os.makedirs(security_dir, exist_ok=True) + with open(os.path.join(security_dir, "location"), "w") as fd: + fd.write(location) + with open(os.path.join(security_dir, "key-type"), "w") as fd: + fd.write(key_type) + + +def _write_legacy_cache(repo_id, previous_location=None): + """Create a legacy-layout ($BORG_CACHE_DIR//chunks marker) cache entry, optionally + with a [cache] previous_location in its config. + """ + cache_path = os.path.join(get_cache_dir(create=False), repo_id) + os.makedirs(cache_path, exist_ok=True) + with open(os.path.join(cache_path, "chunks"), "w") as fd: + fd.write("legacy chunks index placeholder") + if previous_location is not None: + with open(os.path.join(cache_path, "config"), "w") as fd: + fd.write(f"[cache]\nprevious_location = {previous_location}\n") def test_known_repos_empty(archivers, request): @@ -30,6 +43,9 @@ def test_known_repos_empty(archivers, request): result = cmd(archiver, "known-repos") assert "No locally known repositories." in result + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + assert result_json == {"repositories": []} + def test_known_repos_without_repo_argument(archivers, request): """The actual argv operators type - no -r/--repo at all - must work, not just cmd()'s injected one.""" @@ -45,12 +61,6 @@ def test_known_repos_without_repo_argument(archivers, request): assert ids == {repo_id} -def test_known_repos_empty_json(archivers, request): - archiver = request.getfixturevalue(archivers) - result = json.loads(cmd(archiver, "known-repos", "--json")) - assert result == {"repositories": []} - - def test_known_repos_lists_repo_with_cache(archivers, request): archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=1024) @@ -61,11 +71,8 @@ def test_known_repos_lists_repo_with_cache(archivers, request): repo_id = info["id"] result = cmd(archiver, "known-repos") - assert f"Repository ID: {repo_id}" in result - assert "Layout: borg2" in result - assert "Local cache: yes" in result - assert "Cache last written:" in result - assert "Security info: yes (aes256-ocb, sha256)" in result + assert repo_id[:15] in result + assert "borg2" in result result_json = json.loads(cmd(archiver, "known-repos", "--json")) repos = result_json["repositories"] @@ -76,100 +83,86 @@ def test_known_repos_lists_repo_with_cache(archivers, request): assert repo["has_cache"] is True assert repo["cache_size"] > 0 assert repo["key_type"] == "aes256-ocb, sha256" - assert repo["key_type_raw"] is not None assert repo["has_security_info"] is True assert repo["cache_config_mtime"] is not None assert repo["location"] == info["location"] + assert repo["key_file_count"] == 0 # repokey mode (the default) has no key file on disk - -def test_known_repos_without_cache(archivers, request): - """A repo whose local cache was deleted (but security info remains) is still reported, without a cache.""" - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - cache_path = get_cache_dir(repo_id, create=False) - # sanity check: only remove a cache dir that is really below this test's isolated BORG_CACHE_DIR - # (the archiver fixture points BORG_CACHE_DIR at archiver.cache_path, a per-test tmp dir), so a - # misconfigured test environment can not delete a real, unrelated local cache. - assert cache_path.startswith(str(archiver.cache_path) + os.sep) - shutil.rmtree(cache_path, ignore_errors=True) - - result = cmd(archiver, "known-repos") - assert "Local cache: no" in result - assert "Note: a local cache exists" not in result - - result_json = json.loads(cmd(archiver, "known-repos", "--json")) - repos = result_json["repositories"] - assert len(repos) == 1 - repo = repos[0] - assert repo["id"] == repo_id - assert repo["has_cache"] is False - assert repo["cache_size"] is None - assert repo["cache_config_mtime"] is None - # security info (key type, location) must still be reported even without a cache - assert repo["key_type"] == "aes256-ocb, sha256" - assert repo["has_security_info"] is True - assert repo["location"] + # the table's "Last written" column shows the same cache_config_mtime, at/after its header + lines = result.splitlines() + header, row = lines[0], lines[1] + written_col = header.index("Last written") + assert repo["cache_config_mtime"][:10] in row[written_col:] # mtime's date part (YYYY-MM-DD) def test_known_repos_cache_without_security_info(archivers, request): - """A cache dir with no matching security dir entry is still listed, with the "no security info" note.""" + """A cache dir with no matching security dir entry is still listed, with has_security_info False.""" archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] security_path = get_security_dir(repo_id, create=False) - assert security_path.startswith(os.environ["BORG_BASE_DIR"] + os.sep) # same isolation sanity check as above + assert security_path.startswith(os.environ["BORG_BASE_DIR"] + os.sep) shutil.rmtree(security_path, ignore_errors=True) - result = cmd(archiver, "known-repos") - assert f"Repository ID: {repo_id}" in result - assert "Layout: borg2" in result # discovered via the borg2 cache root even with no security info - assert "Local cache: yes" in result - assert "Security info: no" in result - assert "Note: a local cache exists, but no security info was found" in result - result_json = json.loads(cmd(archiver, "known-repos", "--json")) repo = result_json["repositories"][0] - assert repo["layout"] == "borg2" + assert repo["layout"] == "borg2" # discovered via the borg2 cache root even with no security info assert repo["has_cache"] is True assert repo["has_security_info"] is False assert repo["key_type"] is None assert repo["location"] is None -def test_known_repos_location_only_counts_as_security_info(archivers, request): - """A security dir with only a 'location' file (no readable key-type) still counts as having info.""" +def test_known_repos_scans_legacy_v1_security_dir(archivers, request): + """A v1-style security entry (legacy $BORG_CONFIG_DIR/security layout) is listed too, with + no cache: the table's "Last written" column shows "-", not a blank or error. + """ + archiver = request.getfixturevalue(archivers) + legacy_repo_id = "a" * 64 # any syntactically valid 64-hex id; this test never opens a repo + _write_legacy_security(legacy_repo_id, "/some/v1/repo/path") + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + legacy_repo = repos_by_id[legacy_repo_id] + assert legacy_repo["layout"] == "legacy" + assert legacy_repo["location"] == "/some/v1/repo/path" + assert legacy_repo["key_type"] == "aes256-ctr, sha256 (legacy)" + assert legacy_repo["has_cache"] is False + assert legacy_repo["cache_config_mtime"] is None + + table_result = cmd(archiver, "known-repos") + header, row = table_result.splitlines()[0], table_result.splitlines()[1] + written_col = header.index("Last written") + location_col = header.index("Location") + assert row[written_col:location_col].strip() == "-" + + +def test_known_repos_unknown_key_type_is_labeled(archivers, request): + """A key-type byte that does not match any known key class is shown as 'unknown key type N'.""" archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - security_path = get_security_dir(repo_id, create=False) - assert security_path.startswith(os.environ["BORG_BASE_DIR"] + os.sep) - os.remove(os.path.join(security_path, "key-type")) - - result = cmd(archiver, "known-repos") - assert "Security info: yes (key type unknown)" in result - assert "Note: a local cache exists" not in result + key_type_path = os.path.join(get_security_dir(repo_id, create=False), "key-type") + with open(key_type_path, "w") as fd: + fd.write("144") result_json = json.loads(cmd(archiver, "known-repos", "--json")) repo = result_json["repositories"][0] - assert repo["has_security_info"] is True - assert repo["key_type"] is None - assert repo["key_type_raw"] is None - assert repo["location"] + assert repo["key_type"] == "unknown key type 144" def test_known_repos_ignores_storecache_and_non_repo_dirs(archivers, request): - """A non-hex-id cache subdir (e.g. storecache) and a stray security subdir are not listed.""" + """A non-hex-id cache subdir (e.g. storecache), a stray security subdir, and a repo-id- + shaped name with a trailing newline (which must not match the repo-id pattern) are all + ignored. + """ archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] cache_root = get_cache_dir(create=False) - # BORG_STORE_CACHE's storecache dir has no "config" file of its own, but even one placed there - # deliberately must not make it count, since it is not a 64-hex-digit directory name. storecache_dir = f"{cache_root}/storecache" os.makedirs(f"{storecache_dir}/subdir", exist_ok=True) with open(f"{storecache_dir}/config", "w") as fd: @@ -181,6 +174,13 @@ def test_known_repos_ignores_storecache_and_non_repo_dirs(archivers, request): with open(f"{stray_dir}/location", "w") as fd: fd.write("should not be listed") + if os.name != "nt": # Windows rejects a newline in a filename at creation time + bad_name = "a" * 64 + "\n" + bad_dir = os.path.join(security_root, bad_name) + os.makedirs(bad_dir, exist_ok=True) + with open(os.path.join(bad_dir, "location"), "w") as fd: + fd.write("/should/not/be/listed") + result_json = json.loads(cmd(archiver, "known-repos", "--json")) ids = {repo["id"] for repo in result_json["repositories"]} assert ids == {repo_id} @@ -191,7 +191,6 @@ def test_known_repos_multiple_sorted_by_id(archivers, request): cmd(archiver, "repo-create", RK_ENCRYPTION) repo_id1 = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - # a second, independent repository in a different location second_repo = str(archiver.repository_path) + "-second" ret, output = exec_cmd( f"--repo={second_repo}", @@ -218,848 +217,235 @@ def test_known_repos_multiple_sorted_by_id(archivers, request): repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} assert set(repos_by_id) == {repo_id1, repo_id2} assert repos_by_id[repo_id2]["location"] == info2["location"] - # sorted by repository id assert [repo["id"] for repo in result_json["repositories"]] == sorted([repo_id1, repo_id2]) -def test_known_repos_scans_legacy_v1_security_dir(archivers, request): - """A v1-style security entry (legacy $BORG_CONFIG_DIR/security layout) is listed too.""" - archiver = request.getfixturevalue(archivers) - # the archiver fixture isolates BORG_BASE_DIR to a per-test tmp dir, so BORG_CONFIG_DIR - # (unset here) resolves below it via legacy.fs.get_config_dir() - safe to plant a fake entry in. - from borg.legacy.fs import get_config_dir as get_config_dir_legacy - - legacy_repo_id = "a" * 64 # any syntactically valid 64-hex id; this test never opens a repo - legacy_security_dir = os.path.join(get_config_dir_legacy(create=False), "security", legacy_repo_id) - os.makedirs(legacy_security_dir, exist_ok=True) - with open(os.path.join(legacy_security_dir, "location"), "w") as fd: - fd.write("/some/v1/repo/path") - with open(os.path.join(legacy_security_dir, "key-type"), "w") as fd: - fd.write("3") # KeyType.REPO (borg 1.x repokey, legacy AES-CTR) - - result = cmd(archiver, "known-repos") - assert f"Repository ID: {legacy_repo_id}" in result - assert "Layout: legacy" in result - assert "Local cache: no" in result - assert "Security info: yes (aes256-ctr, sha256 (legacy))" in result - - result_json = json.loads(cmd(archiver, "known-repos", "--json")) - repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} - legacy_repo = repos_by_id[legacy_repo_id] - assert legacy_repo["layout"] == "legacy" - assert legacy_repo["location"] == "/some/v1/repo/path" - assert legacy_repo["key_type"] == "aes256-ctr, sha256 (legacy)" - assert legacy_repo["has_cache"] is False - - def test_known_repos_does_not_write_cachedir_tag(archivers, request): - """Running known-repos on an otherwise-empty cache dir must not plant CACHEDIR.TAG there. - - get_cache_dir(create=True) (the default) writes a CACHEDIR.TAG marker file as a side effect; - known-repos uses create=False specifically to avoid that, since it is meant to be read-only. - """ + """Running known-repos on an otherwise-empty cache dir must not plant CACHEDIR.TAG there.""" archiver = request.getfixturevalue(archivers) cache_root = get_cache_dir(create=False) cachedir_tag = os.path.join(cache_root, "CACHEDIR.TAG") - assert not os.path.exists(cachedir_tag) # the archiver fixture's cache dir starts empty + assert not os.path.exists(cachedir_tag) result = cmd(archiver, "known-repos") assert "No locally known repositories." in result assert not os.path.exists(cachedir_tag) -@requires_symlinks -def test_known_repos_does_not_follow_symlinked_cache_entry(archivers, request): - """A cache_root/ entry that is a symlink (not a real directory) must be skipped, not followed. - - Borg itself always creates a plain directory there; a symlink can only come from something else - (deliberate relocation, or an attempt to redirect the read elsewhere) and must not be treated as - a cache. +@pytest.mark.parametrize( + "setup", ["unreadable_security_entry", "security_root_not_a_directory", "unparseable_cache_config"] +) +def test_known_repos_warns_and_exits_nonzero_on_inspection_failure(archivers, request, monkeypatch, setup): + """Any filesystem inspection failure - an EACCES on a repo-id directory, a configured root + that is actually a plain file, or a cache config ConfigParser cannot parse - takes the + same path: it is skipped/treated as absent-looking, but the command still warns and exits + nonzero, rather than crashing or silently looking fully successful. """ archiver = request.getfixturevalue(archivers) - create_regular_file(archiver.input_path, "file1", size=1024) - cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "test", "input") - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - cache_root = get_cache_dir(create=False) - assert cache_root.startswith(str(archiver.cache_path) + os.sep) or cache_root == str(archiver.cache_path) - real_cache_dir = os.path.join(cache_root, repo_id) - moved_cache_dir = real_cache_dir + "-moved" - os.rename(real_cache_dir, moved_cache_dir) - os.symlink(moved_cache_dir, real_cache_dir) - result = cmd(archiver, "known-repos") - assert "Local cache: no" in result - assert "Security info: yes (aes256-ocb, sha256)" in result - - result_json = json.loads(cmd(archiver, "known-repos", "--json")) - repo = result_json["repositories"][0] - assert repo["has_cache"] is False - assert repo["cache_size"] is None - - -@requires_symlinks -def test_known_repos_does_not_follow_symlinked_security_entry(archivers, request): - """A security_root/ entry that is a symlink must be skipped too, same reasoning as the cache one.""" - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - # repo-create also creates a local cache; remove it first so this id has *only* security info, - # all of it reachable only through the symlink this test is about to plant. - cache_path = get_cache_dir(repo_id, create=False) - assert cache_path.startswith(str(archiver.cache_path) + os.sep) - shutil.rmtree(cache_path, ignore_errors=True) - - security_root = get_security_dir(create=False) - real_security_dir = os.path.join(security_root, repo_id) - moved_security_dir = real_security_dir + "-moved" - os.rename(real_security_dir, moved_security_dir) - os.symlink(moved_security_dir, real_security_dir) - - result_json = json.loads(cmd(archiver, "known-repos", "--json")) - ids = {repo["id"] for repo in result_json["repositories"]} - # the id only had security info (via the now-symlinked dir, which must be ignored) and no real - # cache directory, so it must not be reported as a known repository at all. - assert repo_id not in ids + if setup == "unreadable_security_entry": + if os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() == 0: + pytest.skip("chmod-based permission test, not meaningful on Windows or as root") + security_root = get_security_dir(create=False) + repo_id = "d" * 64 + security_path = os.path.join(security_root, repo_id) + os.makedirs(security_path, exist_ok=True) + with open(os.path.join(security_path, "location"), "w") as fd: + fd.write("/should/not/be/readable") + os.chmod(security_path, 0o000) + try: + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be completed" in result + finally: + os.chmod(security_path, 0o755) + return + + if setup == "security_root_not_a_directory": + not_a_dir = str(Path(str(get_cache_dir(create=False))).parent / "not-a-directory") + with open(not_a_dir, "w") as fd: + fd.write("this is a file, not the security directory") + monkeypatch.setenv("BORG_SECURITY_DIR", not_a_dir) + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be completed" in result + return + # setup == "unparseable_cache_config" + repo_id = "f" * 64 + cache_path = os.path.join(get_cache_dir(create=False), repo_id) + os.makedirs(cache_path, exist_ok=True) + with open(os.path.join(cache_path, "chunks"), "w") as fd: + fd.write("legacy chunks index placeholder") + with open(os.path.join(cache_path, "config"), "w") as fd: + fd.write("not a valid ini file [[[\n") + result = cmd(archiver, "known-repos", exit_code=1) + assert "could not be completed" in result -@requires_symlinks -def test_known_repos_cache_only_discovery_does_not_leak_symlinked_security_dir(archivers, request): - """A real cache dir plus a symlinked (attacker-controlled-content) security dir for the same id - must not have that security content read and reported: the id is discovered via its real cache, - but the security_root/ path is a symlink that scan_security_root would reject on its own - - discovering the id through the cache must not grant an implicit pass to read it anyway. +def test_known_repos_legacy_cache_previous_location_fallback(archivers, request): + """A legacy-only cache config's [cache] previous_location is used as location when there is + no security info for that id at all. The location contains a literal '%' (a valid path + character) to also verify it is returned verbatim, not raised on - ConfigParser's default + string interpolation treats '%' specially. """ archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - security_root = get_security_dir(create=False) - real_security_dir = os.path.join(security_root, repo_id) - fake_dir = os.path.join(security_root, "fake-target-for-" + repo_id) - os.makedirs(fake_dir, exist_ok=True) - with open(os.path.join(fake_dir, "location"), "w") as fd: - fd.write("/not/the/real/location") - shutil.rmtree(real_security_dir, ignore_errors=True) - os.symlink(fake_dir, real_security_dir) - - result = cmd(archiver, "known-repos") - assert f"Repository ID: {repo_id}" in result - assert "Local cache: yes" in result - assert "/not/the/real/location" not in result - assert "Security info: no" in result - assert "Note: a local cache exists, but no security info was found" in result + repo_id = "b" * 64 + _write_legacy_cache(repo_id, previous_location="/legacy/100%repo/path") result_json = json.loads(cmd(archiver, "known-repos", "--json")) - repo = next(r for r in result_json["repositories"] if r["id"] == repo_id) - assert repo["location"] is None + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + repo = repos_by_id[repo_id] + assert repo["layout"] == "legacy" + assert repo["location"] == "/legacy/100%repo/path" assert repo["has_security_info"] is False -@requires_symlinks -def test_known_repos_does_not_follow_symlinked_location_file(archivers, request): - """A location file that is itself a symlink (inside an otherwise real, non-symlinked security dir) - - must not be followed either - rejecting a symlinked repo-id *directory* is not enough if a - symlinked *file* inside a real directory can still redirect the read. +def test_known_repos_discovers_legacy_cache_and_keys_in_separate_roots(archiver, monkeypatch, tmp_path): + """A legacy-only cache and key file are still discovered when their roots differ from + the borg2 ones (the macOS default). local-only: the binary archiver subprocess can't see + this monkeypatch. """ - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] + import borg.archiver.known_repos_cmd as known_repos_cmd - security_dir = os.path.join(get_security_dir(create=False), repo_id) - fake_location_file = os.path.join(security_dir, "fake-location.txt") - with open(fake_location_file, "w") as fd: - fd.write("/attacker/leaf/path") - real_location_file = os.path.join(security_dir, "location") - os.remove(real_location_file) - os.symlink(fake_location_file, real_location_file) + separate_cache_root = tmp_path / "legacy-cache-root" + separate_keys_root = tmp_path / "legacy-keys-root" + monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create: str(separate_cache_root)) + monkeypatch.setattr(known_repos_cmd, "get_keys_dir_legacy", lambda create: str(separate_keys_root)) - result = cmd(archiver, "known-repos") - assert "/attacker/leaf/path" not in result - assert "Location: (unknown)" in result + cache_repo_id = "b" * 64 + cache_path = separate_cache_root / cache_repo_id + cache_path.mkdir(parents=True) + (cache_path / "chunks").write_text("legacy chunks index placeholder") + (cache_path / "config").write_text("[cache]\nprevious_location = /legacy/v1/repo/path\n") + + key_repo_id = "f" * 64 + separate_keys_root.mkdir(parents=True) + (separate_keys_root / "keyfile.key").write_text(f"BORG_KEY {key_repo_id}\nrest of the key file\n") result_json = json.loads(cmd(archiver, "known-repos", "--json")) - repo = result_json["repositories"][0] - assert repo["location"] is None + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + + cache_repo = repos_by_id[cache_repo_id] + assert cache_repo["layout"] == "legacy" + assert cache_repo["location"] == "/legacy/v1/repo/path" + assert cache_repo["has_security_info"] is False + key_repo = repos_by_id[key_repo_id] + assert key_repo["key_file_count"] == 1 + assert key_repo["has_cache"] is False -@pytest.mark.skipif(os.name == "nt", reason="Windows rejects a newline in a filename at creation time") -def test_known_repos_rejects_repo_id_with_trailing_newline(archivers, request): - """A directory name of 64 hex digits followed by a newline must not match the repo-id pattern. - re.match with a plain trailing '$' anchor also matches just before a final newline; a directory - actually named "<64 hex>\\n" (an unusual but possible local name) must not be treated as a repo. +@pytest.mark.parametrize( + "legacy_marker", + [ + "chunks", # legacy marker as a plain file + "chunks.archive.d", # legacy marker as a directory (checked first by _cache_layout()) + ], +) +def test_known_repos_layout_undetermined_when_security_roots_coincide(archivers, request, monkeypatch, legacy_marker): + """When $BORG_SECURITY_DIR makes the current and legacy security roots the same path, a + security entry alone cannot tell borg1 and borg2 apart (same "location"/"key-type" + filenames either way) - the cache content (chunks/chunks.archive.d) must decide the layout, + for either form the legacy marker can take. """ archiver = request.getfixturevalue(archivers) security_root = get_security_dir(create=False) - bad_name = "a" * 64 + "\n" - bad_dir = os.path.join(security_root, bad_name) - os.makedirs(bad_dir, exist_ok=True) - with open(os.path.join(bad_dir, "location"), "w") as fd: - fd.write("/should/not/be/listed") - - result_json = json.loads(cmd(archiver, "known-repos", "--json")) - assert result_json["repositories"] == [] + monkeypatch.setenv("BORG_SECURITY_DIR", security_root) - -def test_known_repos_scans_legacy_cache_root_when_different_from_borg2s(request, monkeypatch): - """A v1-only cache (in legacy.fs.get_cache_dir()'s root, not borg2's) must still be listed. - - On platforms where platformdirs (borg2) and the old home-grown layout (legacy.fs) resolve to - different default cache roots (e.g. macOS: "~/Library/Caches/borg" vs "~/.cache/borg"), a v1 - repository's cache lives only in the legacy root. The test fixture sets BORG_BASE_DIR, which - makes both roots resolve to the same place in practice - monkeypatch legacy.fs.get_cache_dir - directly to simulate the platforms where they differ, without depending on the real platform - this test happens to run on. - - Local only (not parametrized over archivers/binary_archiver): the monkeypatch only takes - effect in this pytest process, so it would silently not apply if run against a forked - borg.exe subprocess (binary_archiver) - there is no env var that makes legacy.fs and - helpers.fs resolve to different cache roots to simulate this instead. - """ - archiver = request.getfixturevalue("archiver") cmd(archiver, "repo-create", RK_ENCRYPTION) repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - # move the real cache dir into a separate "legacy" root that differs from the borg2 cache root. - borg2_cache_root = get_cache_dir(create=False) - real_cache_dir = os.path.join(borg2_cache_root, repo_id) - legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") - os.makedirs(legacy_cache_root, exist_ok=True) - os.rename(real_cache_dir, os.path.join(legacy_cache_root, repo_id)) - - import borg.archiver.known_repos_cmd as known_repos_cmd - - monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) - - result = cmd(archiver, "known-repos") - assert f"Repository ID: {repo_id}" in result - assert "Local cache: yes" in result + cache_root = get_cache_dir(create=False) + cache_path = os.path.join(cache_root, repo_id) + marker_path = os.path.join(cache_path, legacy_marker) + if legacy_marker.endswith(".d"): + os.makedirs(marker_path, exist_ok=True) + else: + with open(marker_path, "w") as fd: + fd.write("legacy chunks index placeholder") result_json = json.loads(cmd(archiver, "known-repos", "--json")) repo = result_json["repositories"][0] - assert repo["id"] == repo_id - assert repo["has_cache"] is True - assert repo["cache_size"] > 0 - - -def test_known_repos_warns_and_exits_nonzero_on_oversized_key_type(archivers, request): - """A key-type file too large to interpret must log a warning AND affect the exit code. - - logger.warning() alone does not do that - this command must additionally call - self.print_warning() (see Archiver.print_warning / EXIT_WARNING) so that a script checking the - exit status also learns that the audit was incomplete, not just a human reading stderr. - """ - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - key_type_path = os.path.join(get_security_dir(repo_id, create=False), "key-type") - with open(key_type_path, "w") as fd: - fd.write("x" * 5000) # over the 4096 byte cap in known_repos_cmd._Inspector.read_subfile + assert repo["layout"] == "legacy" - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - assert f"Repository ID: {repo_id}" in result - assert "Security info: yes (key type unknown)" in result - - -def test_known_repos_warns_and_exits_nonzero_on_non_utf8_location(archivers, request): - """A location file that is not valid UTF-8 must be treated like read_subfile returned None: - - warned, counted incomplete, and (since key-type is still readable) reported as - "Security info: yes" without a location, not silently substituted or crashed on. - """ - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - location_path = os.path.join(get_security_dir(repo_id, create=False), "location") - with open(location_path, "wb") as fd: - fd.write(b"\xff\xfe\x00not valid utf-8") - - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - assert f"Repository ID: {repo_id}" in result - assert "Location: (unknown)" in result - assert "Security info: yes" in result - - -def test_known_repos_escapes_control_characters_in_location(archivers, request): - """A location value containing control characters (e.g. from a crafted/corrupted file) must - - be escaped before printing, not written to the terminal raw - _printable() exists precisely - to stop a stored location/key-type value from injecting terminal control sequences. - """ - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - location_path = os.path.join(get_security_dir(repo_id, create=False), "location") - with open(location_path, "w") as fd: - fd.write("/tmp/evil\x1b[31mred\x1b[0m") - - result = cmd(archiver, "known-repos") - assert f"Repository ID: {repo_id}" in result - # the raw ESC byte must not appear unescaped; _printable() renders it as a Python repr - # escape (e.g. "\x1b") instead. - assert "\x1b" not in result - assert "\\x1b" in result - - -def test_known_repos_missing_key_type_file_is_not_a_warning(archivers, request): - """A repo with no key-type file at all (location-only) must not trigger the incomplete-scan warning. - - This is the ordinary "not found" case (see test_known_repos_location_only_counts_as_security_info) - - distinct from "found but could not be read", which does warn. - """ - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - security_path = get_security_dir(repo_id, create=False) - os.remove(os.path.join(security_path, "key-type")) - - result = cmd(archiver, "known-repos") # default exit_code=0 in cmd(); fails if exit code is nonzero - assert "could not be fully completed" not in result - assert f"Repository ID: {repo_id}" in result - - -@requires_chmod_permissions -def test_known_repos_unlistable_cache_subdir_reports_partial_size_not_missing(archivers, request): - """A cache dir that exists, but has a subdirectory that can't be listed, is NOT reported as missing. - - Unlike an unreadable security file (treated the same as absent, see other tests in this file), - an unlistable part of an otherwise-present cache directory still counts as "Local cache: yes" - - just with an incomplete (here: smaller than the real) size, plus a warning and nonzero exit code. - """ - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - cache_dir = get_cache_dir(repo_id, create=False) - # baseline size of whatever repo-create left (config/README/...), before the unlistable subtree. - baseline = 0 - for dirpath, dirnames, filenames in os.walk(cache_dir): - for name in filenames: - baseline += os.path.getsize(os.path.join(dirpath, name)) - - unlistable_subdir = os.path.join(cache_dir, "unlistable") - os.makedirs(unlistable_subdir, exist_ok=True) - with open(os.path.join(unlistable_subdir, "somefile"), "w") as fd: - fd.write("x" * 1000) - os.chmod(unlistable_subdir, 0o000) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - assert "Local cache: yes" in result # not reported as missing, unlike the security-file case - # the unlistable subdir's 1000 bytes are not counted - this is the partial-count behavior - # the epilog describes, not the exact size a healthy scan would report. - assert f"Local cache: yes, {format_file_size(baseline)}" in result - assert "1.00 kB" not in result and "1000 B" not in result - - # --json is not checked here: in the in-process (non-fork) test harness, stdout and stderr - # are the same stream (see exec_cmd()), so the warning line above would land inside what is - # otherwise supposed to be pure JSON output. That collision is a test-harness property that - # also exists for every other command with a --json mode, not something specific to this - # command, and real (forked, non-test) usage keeps the warning on stderr and --json's output - # on stdout separately. - finally: - os.chmod(unlistable_subdir, 0o755) - - -@requires_chmod_permissions -def test_known_repos_unreadable_borg2_cache_does_not_fall_back_to_legacy(request, monkeypatch): - """An unlistable borg2 cache must be reported as 'no cache', and must not publish the legacy - cache's size/mtime for the same id. - - open_cache_entry returns 'unreadable' when the directory is confirmed real but its config - could not be inspected; the command then treats that as "no cache" rather than falling back - to the other root, since the borg2 side's actual state was never established. Mode 0o400 - (readable, not searchable) is used rather than 0o000 so the repo-id directory itself still - opens - only the later stat of its "config" child fails with EACCES, which is the branch - this test targets (0o000 would instead fail at the directory open itself). - """ - archiver = request.getfixturevalue("archiver") - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - borg2_cache_root = get_cache_dir(create=False) - real_cache_dir = os.path.join(borg2_cache_root, repo_id) - legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") - os.makedirs(os.path.join(legacy_cache_root, repo_id), exist_ok=True) - with open(os.path.join(legacy_cache_root, repo_id, "config"), "w") as fd: - fd.write("not-the-borg2-cache") - with open(os.path.join(legacy_cache_root, repo_id, "decoy"), "w") as fd: - fd.write("x" * 5000) - - import borg.archiver.known_repos_cmd as known_repos_cmd - - monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) - - os.chmod(real_cache_dir, 0o400) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - # the borg2 cache's own state is unknown, so this must not claim "yes" (an unverified - # existence claim) or "no" (an unverified absence claim), and must not silently report - # the legacy root's cache either. - assert "Local cache: unknown" in result - # the 5000-byte decoy lives only in the legacy root; "Local cache: unknown" above - # already proves the legacy cache was not substituted (it would have had to report a - # size to do that), so no separate size assertion is needed. --json is not checked - # here: in the in-process (non-fork) test harness, stdout and stderr are the same - # stream, so the warning line above would land inside what is otherwise supposed to - # be pure JSON output (same test-harness property noted elsewhere in this file). - finally: - os.chmod(real_cache_dir, 0o755) - - -@requires_chmod_permissions -def test_known_repos_unsearchable_cache_root_with_nonexistent_child_reports_no_cache(request, monkeypatch): - """An unsearchable cache ROOT (not just one repo-id subdirectory within it) must not make a - - repo-id that never had a cache there look like it has one. When the root itself can't be - opened, cache_root_d is None and kind comes from cache_root_kind ("unreadable") - this - path never reaches open_cache_entry(), since there is no cache_root_d to call it with. - That "unreadable" result must not be treated as has_cache=True, and must not fall back to - a legacy cache either - the borg2 side's existence was never confirmed absent. This is - distinct from the repo-id-directory-level unreadable-cache test above, which makes an - existing repo-id directory (not its parent root) unreadable. - """ - archiver = request.getfixturevalue("archiver") - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - borg2_cache_root = get_cache_dir(create=False) - # remove the normal cache dir for this id entirely - only the unsearchable empty root remains. - shutil.rmtree(os.path.join(borg2_cache_root, repo_id), ignore_errors=True) - - legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") - os.makedirs(os.path.join(legacy_cache_root, repo_id), exist_ok=True) - with open(os.path.join(legacy_cache_root, repo_id, "config"), "w") as fd: - fd.write("not-the-borg2-cache") - import borg.archiver.known_repos_cmd as known_repos_cmd - - monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) - - os.chmod(borg2_cache_root, 0o000) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - # this repo-id was only discovered via the (unreadable) legacy cache root here - its id - # comes from scan_cache_root() on the legacy root reporting it present, while the borg2 - # root's state for it could never be established (root itself unsearchable). The borg2 - # side being unknown must not fabricate "yes", and must not silently claim a confirmed - # "no" either - it surfaces as unknown. --json is not checked (see the test-harness - # note on the previous test in this file). - assert "Local cache: unknown" in result - finally: - os.chmod(borg2_cache_root, 0o755) - - -@requires_chmod_permissions -def test_known_repos_unopenable_security_entry_is_not_dropped(archivers, request): - """A repo-id directory under the security root that list_names() can see, but that this - - command cannot open (EACCES), must still appear in the result - with unknown fields - not - silently disappear from the inventory just because its own inspection failed. Before the - fix for this, scan_security_root()'s `if entry is None: continue` treated "confirmed absent - (missing/symlink)" and "inspection failed (EACCES)" identically, so a name list_names() had - already returned could vanish without a trace in the output - only the summary warning and - exit code hinted anything was wrong, with no way to tell which repository was affected. +def test_known_repos_table_columns_align_with_long_security_label(archivers, request): + """A Security label longer than the old fixed column width (e.g. an "authenticated-*" key + type with the "(legacy)" suffix) must not shift the Location column - the column widths + are computed from the actual data, not hardcoded. """ archiver = request.getfixturevalue(archivers) security_root = get_security_dir(create=False) - repo_id = "d" * 64 + repo_id = "c" * 64 security_path = os.path.join(security_root, repo_id) os.makedirs(security_path, exist_ok=True) with open(os.path.join(security_path, "location"), "w") as fd: - fd.write("/should/not/be/readable") - os.chmod(security_path, 0o000) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - # the repository must still be LISTED - not dropped - with its security/layout fields - # reported as unknown rather than a false "no" or a false "yes". The cache root itself - # was searched successfully and genuinely has no entry for this id, so "Local cache: - # no" here is a real, confirmed answer, not a false negative - this test is only about - # the security-side unreadable entry not causing the repo-id to disappear. --json is - # not checked here (see the in-process stdout/stderr test-harness note elsewhere in - # this file). - assert f"Repository ID: {repo_id}" in result - assert "Layout: unknown" in result - assert "Security info: unknown" in result - assert "Local cache: no" in result - finally: - os.chmod(security_path, 0o755) - - -@requires_chmod_permissions -def test_known_repos_unopenable_cache_entry_is_not_dropped(archivers, request): - """Same as test_known_repos_unopenable_security_entry_is_not_dropped, but for a repo-id - - directory under the cache root that this command cannot open at all (not just whose - "config" child can't be stat'd - see test_known_repos_unreadable_borg2_cache_does_not_fall_back_to_legacy - for that case). scan_cache_root()'s discovery pass must not drop this name either. - """ - archiver = request.getfixturevalue(archivers) - cache_root = get_cache_dir(create=False) - repo_id = "e" * 64 - cache_path = os.path.join(cache_root, repo_id) - os.makedirs(cache_path, exist_ok=True) - with open(os.path.join(cache_path, "config"), "w") as fd: - fd.write("irrelevant") - os.chmod(cache_path, 0o000) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - assert f"Repository ID: {repo_id}" in result - assert "Local cache: unknown" in result - # --json is not checked here (see the in-process stdout/stderr test-harness note - # elsewhere in this file). - finally: - os.chmod(cache_path, 0o755) - - -@requires_chmod_permissions -def test_known_repos_unopenable_cache_entry_with_security_confirmed_absent(archivers, request): - """Same setup as test_known_repos_unopenable_cache_entry_is_not_dropped, but with both - - security roots present and empty (confirmed searched, not just absent by default): - security info must be reported as confirmed absent ("no"), not "unknown". Before the - fix for this, this id fell into do_known_repos()'s catch-all else branch (reached only - via the unreadable-cache discovery path, not borg2_ids/legacy_ids/*_cache_ids), which - unconditionally set has_security_info to None even though the security side had - already been fully and successfully searched. - """ - archiver = request.getfixturevalue(archivers) - # touch both security roots so they exist and are searchable (still empty for this id). - get_security_dir(create=True) - from borg.legacy.fs import get_security_dir as get_security_dir_legacy - - get_security_dir_legacy(create=True) - - cache_root = get_cache_dir(create=False) - repo_id = "f" * 64 - cache_path = os.path.join(cache_root, repo_id) - os.makedirs(cache_path, exist_ok=True) - with open(os.path.join(cache_path, "config"), "w") as fd: - fd.write("irrelevant") - os.chmod(cache_path, 0o000) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert f"Repository ID: {repo_id}" in result - assert "Security info: no" in result - assert "Local cache: unknown" in result - finally: - os.chmod(cache_path, 0o755) + fd.write("/some/repo") + with open(os.path.join(security_path, "key-type"), "w") as fd: + fd.write("7") # KeyType.AUTHENTICATED (legacy) -> "authenticated, sha256 (legacy)", 30 chars + result = cmd(archiver, "known-repos") + lines = result.splitlines() + header, row = lines[0], lines[1] + location_col = header.index("Location") + assert row[location_col:].strip() == "/some/repo" -@requires_chmod_permissions -def test_known_repos_unsearchable_security_parent_warns(archivers, request): - """A security root whose parent cannot be searched must warn, not look empty. - Path.is_dir() on Python 3.14 returns False for this case without raising. A naive - "if not is_dir(): treat as absent" check would therefore exit 0 on an unreadable parent. - The actual code opens the root via os_open() (not Path.is_dir()), which raises EACCES here - and is counted as incomplete via _warn(). +def test_known_repos_counts_keyfile_in_keys_dir(archivers, request): + """A key file in the keys directory is found by its plaintext header (not its filename, + which is a hash of the file's own content) and counted even after security info and the + cache for that repo are both gone - see #10453's "auditing ... keys" use case and + repo-delete, which deletes security info and cache but not the key file. """ archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - security_root = get_security_dir(create=False) - parent = os.path.dirname(security_root) - # only chmod a parent that is inside this test's isolated BORG_BASE_DIR - assert parent.startswith(os.environ["BORG_BASE_DIR"] + os.sep) - os.chmod(parent, 0o000) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - # the repo-create above also left a local cache, so this id is still discovered (via - # the cache root, which is unaffected by chmod'ing the security root's parent) - but - # with the ENTIRE security root unsearchable, "no security entry for this id" can - # never be confirmed; it must surface as unknown, not a false "no". - assert f"Repository ID: {repo_id}" in result - assert "Security info: unknown" in result - finally: - os.chmod(parent, 0o755) - - -@requires_chmod_permissions -def test_known_repos_unreadable_legacy_cache_root_with_confirmed_absent_borg2_cache(request, monkeypatch): - """When the current (borg2) cache is confirmed absent for an id, but the legacy cache - - ROOT ITSELF cannot even be opened (not just a specific entry within it - see - test_known_repos_unreadable_borg2_cache_does_not_fall_back_to_legacy for that case), this - command must not claim "no cache": the legacy side was never actually searched, so the - true answer is unknown. Before the fix for this, the "absent" fallback condition only - checked `legacy_cache_root_d is not None`, which is also true/false independent of whether - the legacy root could be *opened* - if it couldn't be opened, kind simply stayed "absent" - from the already-confirmed-absent borg2 side, silently skipping the (failed) legacy check - instead of surfacing that failure. - """ - archiver = request.getfixturevalue("archiver") - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - # this id has security info (from repo-create) but no borg2 cache entry - delete it so the - # borg2 cache side is confirmed absent, forcing the legacy-cache fallback path. - borg2_cache_root = get_cache_dir(create=False) - shutil.rmtree(os.path.join(borg2_cache_root, repo_id), ignore_errors=True) - - legacy_cache_root = os.path.join(os.path.dirname(borg2_cache_root), "legacy-cache-root") - os.makedirs(legacy_cache_root, exist_ok=True) - - import borg.archiver.known_repos_cmd as known_repos_cmd - - monkeypatch.setattr(known_repos_cmd, "get_cache_dir_legacy", lambda create=False: legacy_cache_root) - - os.chmod(legacy_cache_root, 0o000) - try: - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - assert f"Repository ID: {repo_id}" in result - # borg2 cache: confirmed absent. legacy cache ROOT: could not even be opened - the - # legacy side was never searched, so the combined answer must be unknown, not "no". - assert "Local cache: unknown" in result - finally: - os.chmod(legacy_cache_root, 0o755) - - -def test_known_repos_security_root_listing_failure_does_not_confirm_cache_only_id_absent(request, monkeypatch): - """A security root that opens fine but fails to LIST (e.g. EIO) must not let a - - cache-only id be reported as having confirmed-absent security info - scan_security_root() - cannot have seen every entry under a root it failed to list, so "this id has no security - entry" is not something it can actually assert, even though no individual repo-id - directory's open failed. Before the fix for this, only root-OPEN failure and per-id EACCES - were tracked (security_root_kind / security_unreadable_ids) - a root that opened - successfully and then failed during list_names() itself fell through neither check, so - security_fully_searched stayed True and a cache-only id got a false "no". - """ - archiver = request.getfixturevalue("archiver") - cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "repo-create", RK_ENCRYPTION, "--key-location=keyfile") repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - # this id has a cache (from repo-create) but no security entry for it specifically is - # required - delete the security entry so this id is discovered ONLY via the cache root, - # exercising the borg2_cache_ids branch in do_known_repos(). - security_path = get_security_dir(repo_id, create=False) - shutil.rmtree(security_path, ignore_errors=True) - - import borg.archiver.known_repos_cmd as known_repos_cmd - - orig_list_names = known_repos_cmd._Inspector.list_names - security_root = get_security_dir(create=False) - - def failing_list_names(self, d, display): - # simulate a successful open followed by a listing failure (e.g. EIO) specifically for - # the (already-opened) security root, leaving every other list_names() call (cache - # root, individual repo-id directories) unaffected. - if display == security_root: - raise OSError(5, "Input/output error") # EIO, caught the same way a real one is - return orig_list_names(self, d, display) - - def patched_list_names(self, d, display): - try: - return failing_list_names(self, d, display) - except OSError as exc: - self._warn("known-repos: could not list %s, results may be incomplete: %s", display, exc) - return [] - - monkeypatch.setattr(known_repos_cmd._Inspector, "list_names", patched_list_names) + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = result_json["repositories"][0] + assert repo["id"] == repo_id + assert repo["key_file_count"] == 1 - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - assert f"Repository ID: {repo_id}" in result - # the security root's own listing failed - "no security entry for this id" was never - # actually confirmed, so this must be unknown, not a false "no". - assert "Security info: unknown" in result - # the cache side was unaffected by this failure and should still be confirmed present. - assert "Local cache: yes" in result + cmd(archiver, "repo-delete", "--force") + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + repo = repos_by_id[repo_id] + assert repo["key_file_count"] == 1 + assert repo["has_security_info"] is False + assert repo["has_cache"] is False -@pytest.mark.skipif(os.name == "nt", reason="FIFOs via os.mkfifo are not available on Windows") -def test_known_repos_rejects_fifo_location_without_hanging(archivers, request): - """A stationary FIFO named 'location' (no writer) must be rejected quickly, not hang. +def test_known_repos_ignores_non_key_files_in_keys_dir(archivers, request): + """A file in the keys directory without the BORG_KEY header is not mistaken for a key.""" + from borg.helpers import get_keys_dir - Opening a FIFO read-only blocks until a writer connects; read_subfile must use O_NONBLOCK - on the dir_fd-relative open so this command never stalls on one. - """ archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - security_path = get_security_dir(repo_id, create=False) - location_path = os.path.join(security_path, "location") - os.remove(location_path) - os.mkfifo(location_path) + keys_dir = get_keys_dir(create=True) + create_regular_file(Path(keys_dir), "not_a_key_file", contents=b"just some unrelated file\n") - # cmd() runs in-process (no fork) for the "archiver" param, so a regression of O_NONBLOCK - # here would hang this test rather than fail it cleanly. result = cmd(archiver, "known-repos") - assert f"Repository ID: {repo_id}" in result - assert "Location: (unknown)" in result - - -def test_known_repos_security_only_repo_with_oversized_location_is_still_listed(archivers, request): - """A repo whose ONLY local state is an oversized (unreadable) location file must still be + assert "No locally known repositories." in result - listed with has_security_info True (the file's presence IS confirmed via stat, independent - of whether its content could be decoded) - not disappear, and not be downgraded to "no" - just because the content itself could not be read. - """ - archiver = request.getfixturevalue(archivers) - security_root = get_security_dir(create=False) - repo_id = "c" * 64 - security_path = os.path.join(security_root, repo_id) - os.makedirs(security_path, exist_ok=True) - with open(os.path.join(security_path, "location"), "w") as fd: - fd.write("x" * 5000) # over the 4096 byte cap - result = cmd(archiver, "known-repos", exit_code=1) - assert "could not be fully completed" in result - assert f"Repository ID: {repo_id}" in result - assert "Location: (unknown)" in result - # the oversized location file is this repo's ONLY security state; it was confirmed PRESENT - # via stat (independent of the content read that then failed), so this is "Security info: - # yes" with an unknown key type, not "no" - "no" would be a false claim of absence for a - # file this command already knows exists. - assert "Security info: yes (key type unknown)" in result - - # --json is not checked here: in the in-process (non-fork) test harness, stdout and stderr - # are the same stream, so the warning line above would land inside what is otherwise - # supposed to be pure JSON output (same test-harness property noted elsewhere in this file). - - -def test_known_repos_deep_cache_tree_does_not_hit_recursion_limit(archivers, request): - """dir_size must use an explicit stack, not Python recursion, so a cache directory deeper - - than sys.getrecursionlimit() does not abort the whole command with RecursionError. - - dir_size() holds one fd open per currently-descended level (not per sibling), so a tree - this deep needs that many simultaneously open fds - more than the default soft limit on - macOS (256) or many Linux distros (1024). Raise the soft limit for this test only (never - above the hard limit) rather than reducing the depth, since the depth itself is what - exercises sys.getrecursionlimit(); skip on platforms without an fd-based soft/hard limit - (e.g. Windows, which uses the Path fallback and does not hold one fd per level). +def test_known_repos_flags_location_mismatch_between_security_and_cache(archivers, request): + """A legacy repo whose security info has a location that differs from the cache config's + previous_location is flagged (location_mismatch) rather than silently preferring one over + the other - following the Bash workaround in #10453, which treated this as an error. A + cache previous_location that agrees with the security location is not flagged. """ - if resource is None or not hasattr(resource, "RLIMIT_NOFILE"): - pytest.skip("fd-based open-files limit (resource.RLIMIT_NOFILE) not available on this platform") - depth = 1500 # beyond sys.getrecursionlimit()'s default (~1000) - needed_fds = depth + 64 # +margin for fds pytest/borg itself already holds open - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - if hard != resource.RLIM_INFINITY and hard < needed_fds: - pytest.skip(f"hard open-files limit ({hard}) is below what this depth needs ({needed_fds})") - if soft < needed_fds: - resource.setrlimit(resource.RLIMIT_NOFILE, (needed_fds, hard)) - request.addfinalizer(lambda: resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))) - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - - cache_dir = get_cache_dir(repo_id, create=False) - # build via relative single-component mkdir+chdir steps, not one long os.path.join'd - # absolute path, to avoid hitting the OS's own path-length limit (ENAMETOOLONG) long - # before reaching a depth that would actually exercise Python's recursion limit. - previous_cwd = os.getcwd() - try: - os.chdir(cache_dir) - for d in range(depth): - os.mkdir(f"d{d}") - os.chdir(f"d{d}") - with open("leaf", "w") as fd: - fd.write("x" * 7) - finally: - os.chdir(previous_cwd) + mismatched_id = "c" * 64 + _write_legacy_security(mismatched_id, "/correct/path") + _write_legacy_cache(mismatched_id, previous_location="/stale/different/path") - result = cmd(archiver, "known-repos") - assert f"Repository ID: {repo_id}" in result - assert "Local cache: yes" in result - - -@pytest.mark.skipif(os.name == "nt", reason="test assumes a real fd; Windows always uses the Path fallback") -def test_dir_size_closes_every_handle_when_list_names_raises_base_exception(tmpdir): - """dir_size() must close root (and any already-opened subdirectory) even when list_names() - - raises something that is not an OSError (e.g. KeyboardInterrupt) - including on the very - first call for root, and on the first call for a freshly opened subdirectory, before either - one has been appended to/left on the stack by a completed iteration. A fd leaked here is a - real resource leak, not just a style concern: this is a focused unit test against - _Inspector directly (not the CLI), since the leak is only observable via the raw fd. - """ - from borg.archiver.known_repos_cmd import _Inspector - from pathlib import Path + agreeing_id = "d" * 64 + _write_legacy_security(agreeing_id, "/same/path") + _write_legacy_cache(agreeing_id, previous_location="/same/path") - root = Path(tmpdir) # open_root() is always called with a Path in production code - sub = root / "sub" - sub.mkdir() - (sub / "f.txt").write_bytes(b"x" * 10) - - inspector = _Inspector() - root_fd = inspector.open_root(root, str(root)) - assert isinstance(root_fd, int), "test assumes a real fd, not the Windows Path fallback" - - captured = {} - orig_open_subdir = inspector.open_subdir - - def capturing_open_subdir(parent, name, display): - fd = orig_open_subdir(parent, name, display) - captured["sub_fd"] = fd - return fd - - inspector.open_subdir = capturing_open_subdir - - orig_list_names = inspector.list_names - calls = {"n": 0} - - def flaky_list_names(d, display): - calls["n"] += 1 - if calls["n"] == 2: # the second call is for "sub", opened by the loop below - raise KeyboardInterrupt("simulated interrupt") - return orig_list_names(d, display) - - inspector.list_names = flaky_list_names - - with pytest.raises(KeyboardInterrupt): - inspector.dir_size(root_fd, root) - - with pytest.raises(OSError): - os.fstat(root_fd) - sub_fd = captured.get("sub_fd") - assert isinstance(sub_fd, int) - with pytest.raises(OSError): - os.fstat(sub_fd) - - -@pytest.mark.skipif(os.name == "nt", reason="test assumes a real fd; Windows always uses the Path fallback") -def test_dir_size_closes_root_when_its_own_list_names_raises_base_exception(tmpdir): - """Same leak class as above, but for root's own first list_names() call specifically - - - the one at the very start of dir_size(), before the while loop has run at all. - """ - from borg.archiver.known_repos_cmd import _Inspector - from pathlib import Path - - root = Path(tmpdir) # open_root() is always called with a Path in production code - inspector = _Inspector() - root_fd = inspector.open_root(root, str(root)) - assert isinstance(root_fd, int), "test assumes a real fd, not the Windows Path fallback" - - def raising_list_names(d, display): - raise KeyboardInterrupt("simulated interrupt") - - inspector.list_names = raising_list_names - - with pytest.raises(KeyboardInterrupt): - inspector.dir_size(root_fd, root) + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repos_by_id = {repo["id"]: repo for repo in result_json["repositories"]} + assert repos_by_id[mismatched_id]["location"] == "/correct/path" + assert repos_by_id[mismatched_id]["location_mismatch"] is True + assert repos_by_id[agreeing_id]["location_mismatch"] is False - with pytest.raises(OSError): - os.fstat(root_fd) + table_result = cmd(archiver, "known-repos") + assert "(!= cache's last location)" in table_result diff --git a/src/borg/testsuite/legacy_fs_test.py b/src/borg/testsuite/legacy_fs_test.py index 8dd41d5051..4eb08f38b9 100644 --- a/src/borg/testsuite/legacy_fs_test.py +++ b/src/borg/testsuite/legacy_fs_test.py @@ -12,6 +12,8 @@ get_base_dir as get_base_dir_legacy, get_config_dir as get_config_dir_legacy, get_cache_dir as get_cache_dir_legacy, + get_keys_dir as get_keys_dir_legacy, + get_security_dir as get_security_dir_legacy, ) from ..platform import is_win32, is_darwin, is_haiku, is_cygwin @@ -78,3 +80,32 @@ def test_get_cache_dir_compat(monkeypatch): assert get_cache_dir(create=False) == get_cache_dir_legacy(create=False) monkeypatch.setenv("BORG_CACHE_DIR", "/var/tmp/borg.cache.d") assert get_cache_dir(create=False) == get_cache_dir_legacy(create=False) + + +def test_get_keys_dir_does_not_create_config_dir(monkeypatch, tmp_path): + """get_keys_dir(create=False) must forward create=False to get_config_dir(), not create + the config dir as a side effect (regression). + """ + monkeypatch.delenv("BORG_KEYS_DIR", raising=False) + monkeypatch.delenv("BORG_CONFIG_DIR", raising=False) + monkeypatch.setenv("BORG_BASE_DIR", str(tmp_path)) + + config_dir = tmp_path / ".config" / "borg" + assert not config_dir.exists() + get_keys_dir_legacy(create=False) + assert not config_dir.exists() + + +def test_get_security_dir_does_not_create_config_dir(monkeypatch, tmp_path): + """Same regression as test_get_keys_dir_does_not_create_config_dir, for + get_security_dir(create=False) - what known_repos_cmd.py relies on for its read-only + promise. + """ + monkeypatch.delenv("BORG_SECURITY_DIR", raising=False) + monkeypatch.delenv("BORG_CONFIG_DIR", raising=False) + monkeypatch.setenv("BORG_BASE_DIR", str(tmp_path)) + + config_dir = tmp_path / ".config" / "borg" + assert not config_dir.exists() + get_security_dir_legacy(create=False) + assert not config_dir.exists() From b1ca9eb599d1fbec96b67d5c8b65a90832d47cb0 Mon Sep 17 00:00:00 2001 From: Kazuki Kanda <96773362+KazukiKandaKK@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:06:15 +0900 Subject: [PATCH 3/5] known-repos: fix black formatting, restore generated usage/man docs black wants the process_epilog("""...""") call's opening triple-quote on its own line and the closing paren on a separate line. docs/usage/known-repos.rst still includes known-repos.rst.inc, which a previous commit deleted along with the generated man page - this broke the docs build (InputError: no such file). Regenerate both with scripts/make.py build_usage/build_man, following the convention other recently added commands (copy, find) already use. --- docs/man/borg-known-repos.1 | 91 ++++++++++++++++++++++++++++ docs/usage/known-repos.rst.inc | 55 +++++++++++++++++ src/borg/archiver/known_repos_cmd.py | 6 +- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 docs/man/borg-known-repos.1 create mode 100644 docs/usage/known-repos.rst.inc diff --git a/docs/man/borg-known-repos.1 b/docs/man/borg-known-repos.1 new file mode 100644 index 0000000000..ec4a89219d --- /dev/null +++ b/docs/man/borg-known-repos.1 @@ -0,0 +1,91 @@ +.\" Man page generated from reStructuredText +.\" by the Docutils 0.22.4 manpage writer. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "borg-known-repos" "1" "2026-10-13" "" "borg backup tool" +.SH Name +borg-known-repos \- List repositories Borg has local state for (security info and/or cache). +.SH SYNOPSIS +.sp +borg [common options] known\-repos [options] +.SH DESCRIPTION +.sp +Lists the repositories this local Borg installation currently has state for \- +security info (key type, last known location), a local repository cache, and/or key +files \- without accessing any repository itself (\-r/\-\-repo has no effect). \(dqLayout\(dq +is which state was found in (not the on\-disk repository format, see \(dqborg repo\-info\(dq +for that); \(dqLast written\(dq is the cache config\(aqs mtime, a best\-effort last\-used proxy, +not an exact timestamp and not a staleness check on its own; a mismatch between a +legacy repo\(aqs known location and its cache\(aqs last location is flagged; a key file\(aqs +presence does not by itself mean the repository is still openable. Use \-\-json for a +machine\-readable version, including the full repository id. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS options +.INDENT 0.0 +.TP +.B \-\-json +format output as JSON +.UNINDENT +.SH EXAMPLES +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +$ borg known\-repos +Repository ID Layout Cache Security Key files Last written Location +401606173d54eab.. borg2 180 B aes256\-ocb, sha256 \- 2026\-10\-12T17:15:19 /path/to/repo + +$ borg known\-repos \-\-json +{ + \(dqrepositories\(dq: [ + { + \(dqcache_config_mtime\(dq: \(dq2026\-10\-12T17:15:19.785785+09:00\(dq, + \(dqcache_size\(dq: 180, + \(dqhas_cache\(dq: true, + \(dqhas_security_info\(dq: true, + \(dqid\(dq: \(dq401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb\(dq, + \(dqkey_file_count\(dq: 0, + \(dqkey_type\(dq: \(dqaes256\-ocb, sha256\(dq, + \(dqlayout\(dq: \(dqborg2\(dq, + \(dqlocation\(dq: \(dq/path/to/repo\(dq, + \(dqlocation_mismatch\(dq: false + } + ] +} +.EE +.UNINDENT +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH Author +The Borg Collective +.\" End of generated man page. diff --git a/docs/usage/known-repos.rst.inc b/docs/usage/known-repos.rst.inc new file mode 100644 index 0000000000..9454a62fe6 --- /dev/null +++ b/docs/usage/known-repos.rst.inc @@ -0,0 +1,55 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_known-repos: + +borg known-repos +---------------- +.. code-block:: none + + borg [common options] known-repos [options] + +.. only:: html + + .. class:: borg-options-table + + +-------------------------------------------------------+------------+-----------------------+ + | **options** | + +-------------------------------------------------------+------------+-----------------------+ + | | ``--json`` | format output as JSON | + +-------------------------------------------------------+------------+-----------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+------------+-----------------------+ + + .. raw:: html + + + +.. only:: latex + + + + options + --json format output as JSON + + + :ref:`common_options` + | + +Description +~~~~~~~~~~~ + +Lists the repositories this local Borg installation currently has state for - +security info (key type, last known location), a local repository cache, and/or key +files - without accessing any repository itself (-r/--repo has no effect). "Layout" +is which state was found in (not the on-disk repository format, see "borg repo-info" +for that); "Last written" is the cache config's mtime, a best-effort last-used proxy, +not an exact timestamp and not a staleness check on its own; a mismatch between a +legacy repo's known location and its cache's last location is flagged; a key file's +presence does not by itself mean the repository is still openable. Use --json for a +machine-readable version, including the full repository id. \ No newline at end of file diff --git a/src/borg/archiver/known_repos_cmd.py b/src/borg/archiver/known_repos_cmd.py index dde2c37695..f9c6e51aed 100644 --- a/src/borg/archiver/known_repos_cmd.py +++ b/src/borg/archiver/known_repos_cmd.py @@ -340,7 +340,8 @@ def do_known_repos(self, args): def build_parser_known_repos(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog - known_repos_epilog = process_epilog(""" + known_repos_epilog = process_epilog( + """ Lists the repositories this local Borg installation currently has state for - security info (key type, last known location), a local repository cache, and/or key files - without accessing any repository itself (-r/--repo has no effect). "Layout" @@ -350,7 +351,8 @@ def build_parser_known_repos(self, subparsers, common_parser, mid_common_parser) legacy repo's known location and its cache's last location is flagged; a key file's presence does not by itself mean the repository is still openable. Use --json for a machine-readable version, including the full repository id. - """) + """ + ) subparser = ArgumentParser( parents=[common_parser], description=self.do_known_repos.__doc__, epilog=known_repos_epilog ) From 31cbc3a93a5fa97edf42fe2e8d88286c2dcb03cd Mon Sep 17 00:00:00 2001 From: Kazuki Kanda <96773362+KazukiKandaKK@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:02:29 +0900 Subject: [PATCH 4/5] known-repos: address second-review minor points - location_mismatch now compares the security location and cache previous_location through normalize_local_path(), the same way assert_location_matches() does, so an NFC vs. NFD unicode-form difference alone isn't flagged as a mismatch on macOS (#2913). Added a regression test for it. - The table no longer truncates the repository id to 15 chars - users acting on an entry (e.g. removing a stale cache dir) need the full id. Strengthened the existing test assertion to match. Updated the epilog and the usage example accordingly (full id was previously JSON-only), and regenerated docs/usage/known-repos.rst.inc and docs/man/borg-known-repos.1 to match. - legacy/fs.py: dropped the "(and now also the known-repos command)" historical phrasing from the module docstring; the following paragraph already lists both callers. - Epilog: mentioned that a security-only repo gets layout "unknown" when BORG_SECURITY_DIR makes the borg2 and legacy security roots coincide. Thomas noted the behavior itself is fine either way, just worth a brief mention. --- docs/man/borg-known-repos.1 | 15 ++++++------ docs/usage/known-repos.rst | 4 ++-- docs/usage/known-repos.rst.inc | 11 +++++---- src/borg/archiver/known_repos_cmd.py | 23 +++++++++++------- src/borg/legacy/fs.py | 4 ++-- .../archiver/known_repos_cmd_test.py | 24 ++++++++++++++++++- 6 files changed, 56 insertions(+), 25 deletions(-) diff --git a/docs/man/borg-known-repos.1 b/docs/man/borg-known-repos.1 index ec4a89219d..fa5f0ecc6e 100644 --- a/docs/man/borg-known-repos.1 +++ b/docs/man/borg-known-repos.1 @@ -40,11 +40,12 @@ Lists the repositories this local Borg installation currently has state for \- security info (key type, last known location), a local repository cache, and/or key files \- without accessing any repository itself (\-r/\-\-repo has no effect). \(dqLayout\(dq is which state was found in (not the on\-disk repository format, see \(dqborg repo\-info\(dq -for that); \(dqLast written\(dq is the cache config\(aqs mtime, a best\-effort last\-used proxy, -not an exact timestamp and not a staleness check on its own; a mismatch between a -legacy repo\(aqs known location and its cache\(aqs last location is flagged; a key file\(aqs -presence does not by itself mean the repository is still openable. Use \-\-json for a -machine\-readable version, including the full repository id. +for that) \- a security\-only repo gets \(dqunknown\(dq when $BORG_SECURITY_DIR makes the +borg2 and legacy security roots coincide; \(dqLast written\(dq is the cache config\(aqs mtime, +a best\-effort last\-used proxy, not an exact timestamp and not a staleness check on its +own; a mismatch between a legacy repo\(aqs known location and its cache\(aqs last location +is flagged; a key file\(aqs presence does not by itself mean the repository is still +openable. Use \-\-json for a machine\-readable version. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. @@ -60,8 +61,8 @@ format output as JSON .sp .EX $ borg known\-repos -Repository ID Layout Cache Security Key files Last written Location -401606173d54eab.. borg2 180 B aes256\-ocb, sha256 \- 2026\-10\-12T17:15:19 /path/to/repo +Repository ID Layout Cache Security Key files Last written Location +401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb borg2 180 B aes256\-ocb, sha256 \- 2026\-10\-12T17:15:19 /path/to/repo $ borg known\-repos \-\-json { diff --git a/docs/usage/known-repos.rst b/docs/usage/known-repos.rst index 28558ddb08..c8f518005b 100644 --- a/docs/usage/known-repos.rst +++ b/docs/usage/known-repos.rst @@ -5,8 +5,8 @@ Examples :: $ borg known-repos - Repository ID Layout Cache Security Key files Last written Location - 401606173d54eab.. borg2 180 B aes256-ocb, sha256 - 2026-10-12T17:15:19 /path/to/repo + Repository ID Layout Cache Security Key files Last written Location + 401606173d54eab22cd98de67278d093e3bae7b187da3af8c6430f18fa1112bb borg2 180 B aes256-ocb, sha256 - 2026-10-12T17:15:19 /path/to/repo $ borg known-repos --json { diff --git a/docs/usage/known-repos.rst.inc b/docs/usage/known-repos.rst.inc index 9454a62fe6..9ed08ffcc7 100644 --- a/docs/usage/known-repos.rst.inc +++ b/docs/usage/known-repos.rst.inc @@ -48,8 +48,9 @@ Lists the repositories this local Borg installation currently has state for - security info (key type, last known location), a local repository cache, and/or key files - without accessing any repository itself (-r/--repo has no effect). "Layout" is which state was found in (not the on-disk repository format, see "borg repo-info" -for that); "Last written" is the cache config's mtime, a best-effort last-used proxy, -not an exact timestamp and not a staleness check on its own; a mismatch between a -legacy repo's known location and its cache's last location is flagged; a key file's -presence does not by itself mean the repository is still openable. Use --json for a -machine-readable version, including the full repository id. \ No newline at end of file +for that) - a security-only repo gets "unknown" when $BORG_SECURITY_DIR makes the +borg2 and legacy security roots coincide; "Last written" is the cache config's mtime, +a best-effort last-used proxy, not an exact timestamp and not a staleness check on its +own; a mismatch between a legacy repo's known location and its cache's last location +is flagged; a key file's presence does not by itself mean the repository is still +openable. Use --json for a machine-readable version. \ No newline at end of file diff --git a/src/borg/archiver/known_repos_cmd.py b/src/borg/archiver/known_repos_cmd.py index f9c6e51aed..f51ecfb8bf 100644 --- a/src/borg/archiver/known_repos_cmd.py +++ b/src/borg/archiver/known_repos_cmd.py @@ -6,7 +6,7 @@ from ..crypto.key import LEGACY_KEY_TYPES, AVAILABLE_KEY_TYPES, KEYFILE_ID from ..helpers import format_file_size, json_print, get_security_dir, get_cache_dir, get_keys_dir -from ..helpers import safe_timestamp, OutputTimestamp +from ..helpers import safe_timestamp, OutputTimestamp, normalize_local_path from ..helpers.argparsing import ArgumentParser from ..legacy.fs import get_cache_dir as get_cache_dir_legacy from ..legacy.fs import get_security_dir as get_security_dir_legacy @@ -276,7 +276,13 @@ def do_known_repos(self, args): if location is None: location = cache_location else: - location_mismatch = cache_location is not None and cache_location != location + # Compare the two stored location strings the same way + # assert_location_matches() does: normalize both sides' unicode form + # first (a no-op outside macOS), so on macOS an NFC vs. NFD difference + # alone isn't mistaken for a mismatch (#2913). + location_mismatch = cache_location is not None and normalize_local_path( + cache_location + ) != normalize_local_path(location) repos.append( { @@ -322,7 +328,7 @@ def do_known_repos(self, args): location += " (!= cache's last location)" rows.append( ( - repo["id"][:15] + "..", + repo["id"], repo["layout"] or "unknown", format_file_size(repo["cache_size"]) if repo["has_cache"] else "-", security, @@ -346,11 +352,12 @@ def build_parser_known_repos(self, subparsers, common_parser, mid_common_parser) security info (key type, last known location), a local repository cache, and/or key files - without accessing any repository itself (-r/--repo has no effect). "Layout" is which state was found in (not the on-disk repository format, see "borg repo-info" - for that); "Last written" is the cache config's mtime, a best-effort last-used proxy, - not an exact timestamp and not a staleness check on its own; a mismatch between a - legacy repo's known location and its cache's last location is flagged; a key file's - presence does not by itself mean the repository is still openable. Use --json for a - machine-readable version, including the full repository id. + for that) - a security-only repo gets "unknown" when $BORG_SECURITY_DIR makes the + borg2 and legacy security roots coincide; "Last written" is the cache config's mtime, + a best-effort last-used proxy, not an exact timestamp and not a staleness check on its + own; a mismatch between a legacy repo's known location and its cache's last location + is flagged; a key file's presence does not by itself mean the repository is still + openable. Use --json for a machine-readable version. """ ) subparser = ArgumentParser( diff --git a/src/borg/legacy/fs.py b/src/borg/legacy/fs.py index 0b3693204b..3d3dca53cf 100644 --- a/src/borg/legacy/fs.py +++ b/src/borg/legacy/fs.py @@ -1,5 +1,5 @@ -"""borg 1.x directory layout, kept only so borg transfer (and now also the -known-repos command) can read v1 repo state. +"""borg 1.x directory layout, kept only so borg transfer and known-repos can read v1 +repo state. The borg2 directory functions live in helpers.fs. This module keeps the old home/base, config, cache, keys and security dir resolution. Callers are diff --git a/src/borg/testsuite/archiver/known_repos_cmd_test.py b/src/borg/testsuite/archiver/known_repos_cmd_test.py index 27f2a955ba..79a016203e 100644 --- a/src/borg/testsuite/archiver/known_repos_cmd_test.py +++ b/src/borg/testsuite/archiver/known_repos_cmd_test.py @@ -1,6 +1,7 @@ import json import os import shutil +import unicodedata from pathlib import Path import pytest @@ -71,7 +72,7 @@ def test_known_repos_lists_repo_with_cache(archivers, request): repo_id = info["id"] result = cmd(archiver, "known-repos") - assert repo_id[:15] in result + assert repo_id in result.split() assert "borg2" in result result_json = json.loads(cmd(archiver, "known-repos", "--json")) @@ -449,3 +450,24 @@ def test_known_repos_flags_location_mismatch_between_security_and_cache(archiver table_result = cmd(archiver, "known-repos") assert "(!= cache's last location)" in table_result + + +def test_known_repos_location_mismatch_ignores_nfc_nfd_difference(archivers, request): + """Same path, different unicode normalization (NFC vs. NFD, see #2913) in the security + location and cache previous_location: not a mismatch on macOS (normalize_local_path() + normalizes both), but is one elsewhere (it's a no-op there). + """ + from borg.platformflags import is_darwin + + archiver = request.getfixturevalue(archivers) + nfc_path = unicodedata.normalize("NFC", "/absolute/tëst") + nfd_path = unicodedata.normalize("NFD", "/absolute/tëst") + assert nfc_path != nfd_path # sanity: the two forms really differ byte-wise + + repo_id = "e" * 64 + _write_legacy_security(repo_id, nfc_path) + _write_legacy_cache(repo_id, previous_location=nfd_path) + + result_json = json.loads(cmd(archiver, "known-repos", "--json")) + repo = next(r for r in result_json["repositories"] if r["id"] == repo_id) + assert repo["location_mismatch"] is (not is_darwin) From bc384afd97be9f82a5d88f0ab5c676da61e53a44 Mon Sep 17 00:00:00 2001 From: Kazuki Kanda <96773362+KazukiKandaKK@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:22:52 +0900 Subject: [PATCH 5/5] known-repos: address PyPy msgpack warning and Windows encoding failures Fixes two CI failures reported in maintainer follow-up review: - PyPy: test_known_repos_without_repo_argument and test_known_repos_multiple_sorted_by_id failed with JSONDecodeError because pure-python msgpack emits PURE_PYTHON_MSGPACK_WARNING on PyPy, which contaminated exec_cmd() output. Introduce cmd_without_repo() helper that filters the warning line identically to cmd() while avoiding --repo injection, as suggested. - Windows: test_known_repos_location_mismatch_ignores_nfc_nfd_difference crashed with UnicodeEncodeError writing NFD path in cp1252. Skip the NFD test on Windows, as suggested. - Windows/scanner: read state files using default locale encoding matching borg's SaveFile/SyncFile implementation, dropping explicit encoding="utf-8", as suggested. --- src/borg/archiver/known_repos_cmd.py | 2 +- .../archiver/known_repos_cmd_test.py | 41 ++++++++----------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/borg/archiver/known_repos_cmd.py b/src/borg/archiver/known_repos_cmd.py index f51ecfb8bf..9ea76835e6 100644 --- a/src/borg/archiver/known_repos_cmd.py +++ b/src/borg/archiver/known_repos_cmd.py @@ -94,7 +94,7 @@ def marker_exists(self, path): def read_text(self, path): try: - return path.read_text(encoding="utf-8") + return path.read_text() except FileNotFoundError: return None except OSError as exc: diff --git a/src/borg/testsuite/archiver/known_repos_cmd_test.py b/src/borg/testsuite/archiver/known_repos_cmd_test.py index 79a016203e..fad96e6d8e 100644 --- a/src/borg/testsuite/archiver/known_repos_cmd_test.py +++ b/src/borg/testsuite/archiver/known_repos_cmd_test.py @@ -7,11 +7,19 @@ import pytest from ...helpers import get_cache_dir, get_security_dir -from . import cmd, create_regular_file, exec_cmd, generate_archiver_tests, RK_ENCRYPTION +from . import cmd, create_regular_file, exec_cmd, generate_archiver_tests, RK_ENCRYPTION, PURE_PYTHON_MSGPACK_WARNING pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA +def cmd_without_repo(archiver, *args): + """Like cmd(), but without the injected --repo option.""" + ret, output = exec_cmd(*args, archiver=archiver.archiver, fork=archiver.FORK_DEFAULT, exe=archiver.EXE) + assert ret == 0, output + # same as cmd(): drop the pure-python msgpack warning (e.g. on PyPy), it would break json.loads(). + return "".join(line for line in output.splitlines(keepends=True) if PURE_PYTHON_MSGPACK_WARNING not in line) + + def _write_legacy_security(repo_id, location, key_type="3"): """Create a borg-1.x-style $BORG_CONFIG_DIR/security// entry with the given location and key-type (default "3" == KeyType.REPO, legacy AES-CTR repokey). @@ -54,10 +62,7 @@ def test_known_repos_without_repo_argument(archivers, request): cmd(archiver, "repo-create", RK_ENCRYPTION) repo_id = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] - ret, output = exec_cmd( - "known-repos", "--json", archiver=archiver.archiver, fork=archiver.FORK_DEFAULT, exe=archiver.EXE - ) - assert ret == 0, output + output = cmd_without_repo(archiver, "known-repos", "--json") ids = {repo["id"] for repo in json.loads(output)["repositories"]} assert ids == {repo_id} @@ -193,25 +198,8 @@ def test_known_repos_multiple_sorted_by_id(archivers, request): repo_id1 = json.loads(cmd(archiver, "repo-info", "--json"))["repository"]["id"] second_repo = str(archiver.repository_path) + "-second" - ret, output = exec_cmd( - f"--repo={second_repo}", - "repo-create", - RK_ENCRYPTION, - archiver=archiver.archiver, - fork=archiver.FORK_DEFAULT, - exe=archiver.EXE, - ) - assert ret == 0, output - ret, output = exec_cmd( - f"--repo={second_repo}", - "repo-info", - "--json", - archiver=archiver.archiver, - fork=archiver.FORK_DEFAULT, - exe=archiver.EXE, - ) - assert ret == 0, output - info2 = json.loads(output)["repository"] + cmd_without_repo(archiver, f"--repo={second_repo}", "repo-create", RK_ENCRYPTION) + info2 = json.loads(cmd_without_repo(archiver, f"--repo={second_repo}", "repo-info", "--json"))["repository"] repo_id2 = info2["id"] result_json = json.loads(cmd(archiver, "known-repos", "--json")) @@ -457,7 +445,10 @@ def test_known_repos_location_mismatch_ignores_nfc_nfd_difference(archivers, req location and cache previous_location: not a mismatch on macOS (normalize_local_path() normalizes both), but is one elsewhere (it's a no-op there). """ - from borg.platformflags import is_darwin + from borg.platformflags import is_darwin, is_win32 + + if is_win32: + pytest.skip("NFD path is not encodable in the Windows locale encoding") archiver = request.getfixturevalue(archivers) nfc_path = unicodedata.normalize("NFC", "/absolute/tëst")