diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0f36aa368..f12e5d71b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: asan_ubsan: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 25 needs: [lint] @@ -73,7 +73,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: '3.14' - name: Install system packages run: | @@ -141,21 +141,13 @@ jobs: ${{ fromJSON( github.event_name == 'pull_request' && '{ "include": [ - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "mypy"}, - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "docs"}, - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, - {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"} + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "mypy"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "docs"}, + {"os": "ubuntu-26.04", "python-version": "3.14", "toxenv": "py314-mfusepy"} + {"os": "macos-15", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-arm64-gh"}, ] }' || '{ "include": [ - {"os": "ubuntu-24.04", "python-version": "3.11", "toxenv": "py311-llfuse"}, - {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"}, - {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-mfusepy"}, - {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc239-x86_64-gh"}, - {"os": "ubuntu-24.04-arm", "python-version": "3.14", "toxenv": "py314-pyfuse3", "binary": "borg-linux-glibc239-arm64-gh"}, - {"os": "macos-15", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-arm64-gh"}, - {"os": "macos-15-intel", "python-version": "3.14", "toxenv": "py314-none", "binary": "borg-macos-15-x86_64-gh"} ] }' ) }} @@ -392,16 +384,11 @@ jobs: matrix: include: - os: freebsd - version: '14.3' + version: '15.0' display_name: FreeBSD # Controls binary build and provenance attestation on tags do_binaries: true - artifact_prefix: borg-freebsd-14-x86_64-gh - - - os: netbsd - version: '10.1' - display_name: NetBSD - do_binaries: false + artifact_prefix: borg-freebsd-15-x86_64-gh - os: openbsd version: '7.8' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e02fc6add8..08ad5537df 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ concurrency: jobs: analyze: name: Analyze - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 20 permissions: actions: read @@ -53,7 +53,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: 3.11 + python-version: 3.14 - name: Cache pip uses: actions/cache@v5 with: diff --git a/docs/changes.rst b/docs/changes.rst index 566c3d4610..9617a6843b 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -168,6 +168,7 @@ above. New features: +- monitor: access encrypted/signed monitoring data in the repository, #9788 - repo-create: split ``--encryption`` into orthogonal options. ``--encryption`` now selects only the cipher / AE algorithm (``none``, ``authenticated``, ``aes256-ocb`` or ``chacha20-poly1305``), the new ``--id-hash`` selects the id hash function diff --git a/docs/usage.rst b/docs/usage.rst index 6921d0bc7c..c1d36a941a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -41,6 +41,7 @@ Usage usage/repo-list usage/repo-info usage/repo-delete + usage/monitor usage/serve usage/version usage/compact diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc index 120025979c..8bddc95307 100644 --- a/docs/usage/general/environment.rst.inc +++ b/docs/usage/general/environment.rst.inc @@ -256,6 +256,13 @@ Directories and files: - the key file must either exist (for most commands) or will be created (``borg repo-create``). - you need to give a different path for different repositories. - you need to point to the correct key file matching the repository the command will operate on. + BORG_MONITORING_KEY + Used by ``borg monitor`` on the monitoring host to verify and decrypt + the reports backup clients published into the repository. + You can get the correct key value by running ``borg monitor --key`` on + a host that has access to the borg key. + This key can not be used to create reports, nor does it grant access + to backup data. TMPDIR This is where temporary files are stored (might need a lot of temporary space for some operations), see tempfile_ for details. diff --git a/docs/usage/monitor.rst b/docs/usage/monitor.rst new file mode 100644 index 0000000000..3a9b6e1e66 --- /dev/null +++ b/docs/usage/monitor.rst @@ -0,0 +1 @@ +.. include:: monitor.rst.inc diff --git a/docs/usage/monitor.rst.inc b/docs/usage/monitor.rst.inc new file mode 100644 index 0000000000..110714910c --- /dev/null +++ b/docs/usage/monitor.rst.inc @@ -0,0 +1,88 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_monitor: + +borg monitor +------------ +.. code-block:: none + + borg [common options] monitor [options] + +.. only:: html + + .. class:: borg-options-table + + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | **options** | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--key`` | derive and print BORG_MONITORING_KEY for this repository (needs the borg key) | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--name SERIES`` | only report on the given archive series (e.g. the name used with borg create) | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--command COMMAND`` | only report on the given command, e.g. create or prune (default: all commands) | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--host HOSTNAME`` | only report on backups made from the given host | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--user USERNAME`` | only report on backups made by the given user | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--max-age SECONDS`` | freshness window in seconds; older reports count as stale (default: 90000) | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--keep N`` | after reading, delete all but the N newest report objects (0 = do not clean up; default: 500) | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | | ``--json`` | format output as JSON | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------------------+ + + .. raw:: html + + + +.. only:: latex + + + + options + --key derive and print BORG_MONITORING_KEY for this repository (needs the borg key) + --name SERIES only report on the given archive series (e.g. the name used with borg create) + --command COMMAND only report on the given command, e.g. create or prune (default: all commands) + --host HOSTNAME only report on backups made from the given host + --user USERNAME only report on backups made by the given user + --max-age SECONDS freshness window in seconds; older reports count as stale (default: 90000) + --keep N after reading, delete all but the N newest report objects (0 = do not clean up; default: 500) + --json format output as JSON + + + :ref:`common_options` + | + +Description +~~~~~~~~~~~ + +Read trusted monitoring state of a repository. + +Borg client commands publish a signed-and-encrypted state report into the +repository after each run. Only borg monitor can read these reports using +the monitoring key. + +Setup (once, on a host that has the borg key):: + + borg monitor --key # this outputs the monitoring key + +Then, on the monitoring host:: + + BORG_MONITORING_KEY= borg monitor + +This verifies and decrypts the reports and prints, for each distinct backup job +(host, user, command, archive series), the latest status and its age. It exits with +a non-zero code (warning or error) if any job is missing, stale (older than +--max-age), unsigned, or did not indicate success - so it can drive alerting like a +dead man's switch. Use --name, --command, --host and --user to restrict the output. +Reports accumulate over time; --keep=N (default 500) deletes all but the N newest +after reading. \ No newline at end of file diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index 833b5d821c..69cca8a04c 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -84,6 +84,7 @@ from .recreate_cmd import RecreateMixIn from .rename_cmd import RenameMixIn from .repo_create_cmd import RepoCreateMixIn +from .monitor_cmd import MonitorMixIn from .repo_info_cmd import RepoInfoMixIn from .repo_delete_cmd import RepoDeleteMixIn from .repo_list_cmd import RepoListMixIn @@ -112,6 +113,7 @@ class Archiver( KeysMixIn, ListMixIn, LocksMixIn, + MonitorMixIn, MountMixIn, PruneMixIn, RecreateMixIn, @@ -301,6 +303,7 @@ def build_parser(self): self.build_parser_keys(subparsers, common_parser, mid_common_parser) self.build_parser_list(subparsers, common_parser, mid_common_parser) self.build_parser_locks(subparsers, common_parser, mid_common_parser) + self.build_parser_monitor(subparsers, common_parser, mid_common_parser) self.build_parser_mount_umount(subparsers, common_parser, mid_common_parser) self.build_parser_prune(subparsers, common_parser, mid_common_parser) self.build_parser_repo_create(subparsers, common_parser, mid_common_parser) diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 44cb4fbc23..be23188173 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -10,6 +10,7 @@ from ._common import with_repository, Highlander from .. import helpers +from .. import monitoring from ..archive import Archive, is_special, SF_DATALESS from ..archive import BackupError, BackupOSError, BackupItemExcluded, backup_io, OsOpen, stat_update_check from ..archive import FilesystemObjectProcessors, MetadataCollector, ChunksProcessor @@ -287,7 +288,18 @@ def create_inner(archive, cache, fso): files_changed=args.files_changed, ) create_inner(archive, cache, fso) - else: + + monitoring.publish_command_report( + repository, + manifest.key, + "create", + hostname=archive.hostname, + username=archive.username, + archive=archive.name, + archive_id=archive.id, + stats=archive.stats.as_dict(), + ) + else: # dry-run create_inner(None, None, None) def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, dry_run, strip_prefix): diff --git a/src/borg/archiver/delete_cmd.py b/src/borg/archiver/delete_cmd.py index 985abb475c..d489fe1acd 100644 --- a/src/borg/archiver/delete_cmd.py +++ b/src/borg/archiver/delete_cmd.py @@ -1,6 +1,7 @@ import logging from ._common import with_repository +from .. import monitoring from ..constants import * # NOQA from ..helpers import format_archive, CommandError, bin_to_hex, archivename_validator from ..helpers.argparsing import ArgumentParser @@ -32,7 +33,7 @@ def do_delete(self, args, repository): "or just delete the whole repository (might be much faster)." ) - deleted = False + deleted_count = 0 logger_list = logging.getLogger("borg.output.list") for i, archive_info in enumerate(archive_infos, 1): name, id, hex_id = archive_info.name, archive_info.id, bin_to_hex(archive_info.id) @@ -46,17 +47,24 @@ def do_delete(self, args, repository): except KeyError: self.print_warning(f"Archive {name} {hex_id} not found ({i}/{count}).") else: - deleted = True + deleted_count += 1 if self.output_list: msg = "Would delete: {} ({}/{})" if dry_run else "Deleted archive: {} ({}/{})" logger_list.info(msg.format(archive_formatted, i, count)) if dry_run: logger.info("Finished dry-run.") - elif deleted: + elif deleted_count: manifest.write() self.print_warning('Done. Run "borg compact" to free space.', wc=None) else: self.print_warning("Aborted.", wc=None) + if not dry_run: + monitoring.publish_command_report( + repository, + manifest.key, + "delete", + stats={"archives_deleted": deleted_count, "archives_considered": count}, + ) return def build_parser_delete(self, subparsers, common_parser, mid_common_parser): diff --git a/src/borg/archiver/monitor_cmd.py b/src/borg/archiver/monitor_cmd.py new file mode 100644 index 0000000000..75c7349809 --- /dev/null +++ b/src/borg/archiver/monitor_cmd.py @@ -0,0 +1,243 @@ +import os +from datetime import datetime, timezone + +from ._common import with_repository +from ..constants import * # NOQA +from ..helpers import set_ec, Error, json_print +from ..helpers.time import parse_timestamp +from ..crypto.low_level import IntegrityError +from ..helpers.argparsing import ArgumentParser +from ..manifest import Manifest +from .. import monitoring +from ..crypto import monitoring as mon_crypto + +from ..logger import create_logger + +logger = create_logger() + +# Default freshness window: alert if the newest report is older than this (slightly over +# a day, to tolerate a late daily backup). Override with --max-age. +DEFAULT_MAX_AGE = 25 * 3600 + + +class MonitorMixIn: + @with_repository(manifest=False) + def do_monitor(self, args, repository): + """Read or export trusted monitoring state of a repository. + + Without arguments this reads the monitoring reports that backup-side commands + published into the repository, verifies and decrypts them using the key from the + BORG_MONITORING_KEY environment variable, and reports the latest status and + freshness of each distinct backup job - identified by host, user, command and + archive series. Because each job is reported independently, a later successful + backup does not mask an earlier failed one - not even when several hosts back up + the same archive series name to one repository. Restrict the output with --name + (archive series), --command (e.g. create or prune), --host and --user. Neither the + repository passphrase nor the borg key is needed for reading. + + Reports accumulate over time; --keep=N deletes all but the N newest after reading. + + With --key (which does need the borg key), it derives and prints the + BORG_MONITORING_KEY value for this repository, to be configured on the monitoring + host. The printed value only allows verifying and decrypting reports, not creating + them. + """ + if args.key: + return self._monitor_export_key(repository) + return self._monitor_read(args, repository) + + def _monitor_export_key(self, repository): + manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + if not mon_crypto.is_signed_repo(manifest.key): + raise Error( + "This repository is unencrypted; monitoring reports are not signed, so there is no monitoring key." + ) + print(mon_crypto.export_monitor_key(manifest.key)) + + def _monitor_read(self, args, repository): + key_str = os.environ.get("BORG_MONITORING_KEY") + try: + monitor_key = mon_crypto.parse_monitor_key(key_str) if key_str else None + except ValueError as e: + raise Error(f"Invalid BORG_MONITORING_KEY: {e}") + + try: + reports = list(monitoring.iter_reports(repository, monitor_key)) + except IntegrityError as e: + raise Error(f"Monitoring report verification/decryption failed (wrong key or tampered): {e}") + except ValueError as e: + raise Error(str(e)) + + # Cleanup is best-effort and must not change the read result, so do it after reading. + monitoring.prune_reports(repository, args.keep) + + now = datetime.now(timezone.utc) + + # Group by (host, user, command, archive). Reports are oldest-first, so the newest + # one per group wins - giving each distinct backup job its latest status + # independently. Including host/user means two hosts backing up the same archive + # series name (to the same repo) do not mask each other. + latest = {} + for report, trusted in reports: + if args.command and report.get("command") != args.command: + continue + if args.name and report.get("archive") != args.name: + continue + if args.host and report.get("hostname") != args.host: + continue + if args.user and report.get("username") != args.user: + continue + key = (report.get("hostname"), report.get("username"), report.get("command"), report.get("archive")) + latest[key] = (report, trusted) + + entries = [] + for key in sorted(latest, key=lambda k: tuple("" if v is None else str(v) for v in k)): + report, trusted = latest[key] + age = (now - parse_timestamp(report["time"])).total_seconds() + entries.append({"report": report, "trusted": trusted, "age": age, "stale": age > args.max_age}) + + self._monitor_output(args, entries) + + if not entries: + set_ec(EXIT_ERROR) # nothing matched -> dead man's switch fires + return + # Exit code drives external alerting: worst job wins (stale/error -> error, + # warning/untrusted -> warning). + for e in entries: + if e["stale"] or e["report"].get("status") == "error": + set_ec(EXIT_ERROR) + elif e["report"].get("status") == "warning" or not e["trusted"]: + set_ec(EXIT_WARNING) + + @staticmethod + def _unit_label(report): + who = f"{report.get('username', '?')}@{report.get('hostname', '?')}" + return f"{who} {report.get('archive') or report.get('command')}" + + def _monitor_output(self, args, entries): + if args.json: + out = { + "max_age_seconds": args.max_age, + "entries": [ + { + "hostname": e["report"].get("hostname"), + "username": e["report"].get("username"), + "command": e["report"].get("command"), + "archive": e["report"].get("archive"), + "trusted": e["trusted"], + "stale": e["stale"], + "age_seconds": e["age"], + "report": e["report"], + } + for e in entries + ], + } + json_print(out) + return + if not entries: + filters = [ + f"{k} '{v}'" + for k, v in ( + ("archive", args.name), + ("command", args.command), + ("host", args.host), + ("user", args.user), + ) + if v + ] + scope = (" for " + ", ".join(filters)) if filters else "" + print(f"No monitoring report found{scope}.") + return + for e in entries: + report = e["report"] + print(f"{self._unit_label(report)}:") + print(f" command: {report.get('command')}") + print(f" status: {report.get('status')} (rc {report.get('rc')})") + print(f" user@host: {report.get('username', '-')}@{report.get('hostname', '-')}") + print(f" archive: {report.get('archive', '-')}") + print(f" time: {report.get('time')}") + print(f" age: {int(e['age'])}s (max {args.max_age}s){' STALE' if e['stale'] else ''}") + print(f" trusted: {e['trusted']}{'' if e['trusted'] else ' (unsigned - repo is unencrypted)'}") + + def build_parser_monitor(self, subparsers, common_parser, mid_common_parser): + from ._common import process_epilog + + monitor_epilog = process_epilog( + """ + Read trusted monitoring state of a repository. + + Borg client commands publish a signed-and-encrypted state report into the + repository after each run. Only borg monitor can read these reports using + the monitoring key. + + Setup (once, on a host that has the borg key):: + + borg monitor --key # this outputs the monitoring key + + Then, on the monitoring host:: + + BORG_MONITORING_KEY= borg monitor + + This verifies and decrypts the reports and prints, for each distinct backup job + (host, user, command, archive series), the latest status and its age. It exits with + a non-zero code (warning or error) if any job is missing, stale (older than + --max-age), unsigned, or did not indicate success - so it can drive alerting like a + dead man's switch. Use --name, --command, --host and --user to restrict the output. + Reports accumulate over time; --keep=N (default 500) deletes all but the N newest + after reading. + """ + ) + subparser = ArgumentParser(parents=[common_parser], description=self.do_monitor.__doc__, epilog=monitor_epilog) + subparsers.add_subcommand("monitor", subparser, help="read/export repository monitoring state") + subparser.add_argument( + "--key", + dest="key", + action="store_true", + help="derive and print BORG_MONITORING_KEY for this repository (needs the borg key)", + ) + subparser.add_argument( + "--name", + dest="name", + default=None, + metavar="SERIES", + help="only report on the given archive series (e.g. the name used with borg create)", + ) + subparser.add_argument( + "--command", + dest="command", + default=None, + metavar="COMMAND", + help="only report on the given command, e.g. create or prune (default: all commands)", + ) + subparser.add_argument( + "--host", + dest="host", + default=None, + metavar="HOSTNAME", + help="only report on backups made from the given host", + ) + subparser.add_argument( + "--user", + dest="user", + default=None, + metavar="USERNAME", + help="only report on backups made by the given user", + ) + subparser.add_argument( + "--max-age", + dest="max_age", + type=int, + default=DEFAULT_MAX_AGE, + metavar="SECONDS", + help=f"freshness window in seconds; older reports count as stale (default: {DEFAULT_MAX_AGE})", + ) + subparser.add_argument( + "--keep", + dest="keep", + type=int, + default=monitoring.DEFAULT_KEEP, + metavar="N", + help="after reading, delete all but the N newest report objects " + f"(0 = do not clean up; default: {monitoring.DEFAULT_KEEP})", + ) + subparser.add_argument("--json", action="store_true", help="format output as JSON") diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index dfa9793fc9..12d1b73010 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -4,6 +4,7 @@ from operator import attrgetter import os +from .. import monitoring from ._common import with_repository, Highlander from ..constants import * # NOQA from ..helpers import ArchiveFormatter, interval, sig_int, ProgressIndicatorPercent, CommandError, Error @@ -232,6 +233,18 @@ def do_prune(self, args, repository, manifest): if sig_int: raise Error("Got Ctrl-C / SIGINT.") + if not args.dry_run: + monitoring.publish_command_report( + repository, + manifest.key, + "prune", + stats={ + "archives_pruned": archives_deleted, + "archives_kept": len(keep), + "archives_considered": len(archives), + }, + ) + def build_parser_prune(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog from ._common import define_archive_filters_group diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index d51e6261f0..295c1b5a52 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -1,4 +1,5 @@ from ._common import with_repository, with_other_repository, Highlander +from .. import monitoring from ..archive import Archive, cached_hash, DownloadPipeline from ..chunkers import get_chunker from ..constants import * # NOQA @@ -186,6 +187,9 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non upgrader = UpgraderCls(cache=cache, args=args) + transferred_count = 0 + skipped_count = 0 + transferred_size = 0 for archive_info in archive_infos: name, id, ts = archive_info.name, archive_info.id, archive_info.ts id_hex, ts_str = bin_to_hex(id), ts.isoformat() @@ -201,9 +205,11 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non # Useful for Borg 1.x -> 2 transfers; we have unique names in Borg 1.x. # Also useful for Borg 2 -> 2 transfers with metadata changes (ID changes). print(f"{name} {ts_str}: archive is already present in destination repo, skipping.") + skipped_count += 1 elif not dry_run and manifest.archives.exists_name_and_id(name, id): # Useful for Borg 2 -> 2 transfers without changes (ID stays the same) print(f"{name} {id_hex}: archive is already present in destination repo, skipping.") + skipped_count += 1 else: if not dry_run: print(f"{name} {ts_str} {id_hex}: copying archive to destination repo...") @@ -253,6 +259,8 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non archive.stats.show_progress(final=True) additional_metadata = upgrader.upgrade_archive_metadata(metadata=other_archive.metadata) archive.save(additional_metadata=additional_metadata) + transferred_count += 1 + transferred_size += transfer_size print( f"{name} {ts_str} {id_hex}: finished. " f"transfer_size: {format_file_size(transfer_size)} " @@ -267,6 +275,19 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non f"present_size: {format_file_size(present_size)}" ) + if not dry_run: + monitoring.publish_command_report( + repository, + manifest.key, + "transfer", + stats={ + "archives_transferred": transferred_count, + "archives_skipped": skipped_count, + "archives_considered": count, + "transferred_size": transferred_size, + }, + ) + def build_parser_transfer(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog from ._common import define_archive_filters_group diff --git a/src/borg/archiver/undelete_cmd.py b/src/borg/archiver/undelete_cmd.py index 8a37e6fb5f..7b96821aa6 100644 --- a/src/borg/archiver/undelete_cmd.py +++ b/src/borg/archiver/undelete_cmd.py @@ -1,6 +1,7 @@ import logging from ._common import with_repository +from .. import monitoring from ..constants import * # NOQA from ..helpers import format_archive, CommandError, bin_to_hex, archivename_validator from ..helpers.argparsing import ArgumentParser @@ -29,7 +30,7 @@ def do_undelete(self, args, repository): if not args.name and not args.match_archives and args.first == 0 and args.last == 0: raise CommandError("Aborting: if you really want to undelete all archives, please use -a 'sh:*'.") - undeleted = False + undeleted_count = 0 logger_list = logging.getLogger("borg.output.list") for i, archive_info in enumerate(archive_infos, 1): name, id, hex_id = archive_info.name, archive_info.id, bin_to_hex(archive_info.id) @@ -39,17 +40,24 @@ def do_undelete(self, args, repository): except KeyError: self.print_warning(f"Archive {name} {hex_id} not found ({i}/{count}).") else: - undeleted = True + undeleted_count += 1 if self.output_list: msg = "Would undelete: {} ({}/{})" if dry_run else "Undeleted archive: {} ({}/{})" logger_list.info(msg.format(format_archive(archive_info), i, count)) if dry_run: logger.info("Finished dry-run.") - elif undeleted: + elif undeleted_count: manifest.write() self.print_warning("Done.", wc=None) else: self.print_warning("Aborted.", wc=None) + if not dry_run: + monitoring.publish_command_report( + repository, + manifest.key, + "undelete", + stats={"archives_undeleted": undeleted_count, "archives_considered": count}, + ) return def build_parser_undelete(self, subparsers, common_parser, mid_common_parser): diff --git a/src/borg/crypto/low_level.pyi b/src/borg/crypto/low_level.pyi index 9d5e4c0566..c1ee6afa19 100644 --- a/src/borg/crypto/low_level.pyi +++ b/src/borg/crypto/low_level.pyi @@ -13,6 +13,20 @@ def hmac_sha256(key: bytes, data: bytes) -> bytes: ... def blake2b_256(key: bytes, data: bytes) -> bytes: ... def blake2b_128(data: bytes) -> bytes: ... +# Asymmetric primitives for monitoring reports (OpenSSL >= 3.2) +ED25519_SEED_SIZE: int +ED25519_PUBLIC_SIZE: int +ED25519_SIGNATURE_SIZE: int +X25519_SEED_SIZE: int +X25519_PUBLIC_SIZE: int + +def ed25519_public_from_seed(seed: bytes) -> bytes: ... +def x25519_public_from_seed(seed: bytes) -> bytes: ... +def ed25519_sign(seed: bytes, data: bytes) -> bytes: ... +def ed25519_verify(public: bytes, data: bytes, signature: bytes) -> None: ... +def hpke_seal(recipient_public: bytes, info: bytes, aad: bytes, plaintext: bytes) -> bytes: ... +def hpke_open(recipient_secret: bytes, info: bytes, aad: bytes, blob: bytes) -> bytes: ... + # Exception classes class CryptoError(Exception): """Malfunction in the crypto module.""" diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 149ea1d738..2ffb84cbbd 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -42,7 +42,7 @@ from cpython cimport PyMem_Malloc, PyMem_Free from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AsString from libc.stdlib cimport malloc, free -from libc.stdint cimport uint8_t, uint32_t, uint64_t +from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t from libc.string cimport memset, memcpy @@ -89,6 +89,66 @@ cdef extern from "openssl/evp.h": int EVP_CTRL_AEAD_SET_IVLEN +cdef extern from "openssl/evp.h": + # asymmetric keys (Ed25519 signing, X25519 key agreement for HPKE) + ctypedef struct EVP_PKEY: + pass + ctypedef struct EVP_PKEY_CTX: + pass + ctypedef struct EVP_MD_CTX: + pass + + int EVP_PKEY_ED25519 + int EVP_PKEY_X25519 + + EVP_PKEY *EVP_PKEY_new_raw_private_key(int type, ENGINE *e, const unsigned char *key, size_t keylen) + EVP_PKEY *EVP_PKEY_new_raw_public_key(int type, ENGINE *e, const unsigned char *key, size_t keylen) + int EVP_PKEY_get_raw_public_key(const EVP_PKEY *pkey, unsigned char *pub, size_t *len) + void EVP_PKEY_free(EVP_PKEY *key) + + EVP_MD_CTX *EVP_MD_CTX_new() + void EVP_MD_CTX_free(EVP_MD_CTX *ctx) + int EVP_DigestSignInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey) + int EVP_DigestSign(EVP_MD_CTX *ctx, unsigned char *sig, size_t *siglen, const unsigned char *tbs, size_t tbslen) + int EVP_DigestVerifyInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey) + int EVP_DigestVerify(EVP_MD_CTX *ctx, const unsigned char *sig, size_t siglen, const unsigned char *tbs, size_t tbslen) + + +cdef extern from "openssl/hpke.h": + ctypedef struct OSSL_HPKE_SUITE: + uint16_t kem_id + uint16_t kdf_id + uint16_t aead_id + ctypedef struct OSSL_HPKE_CTX: + pass + ctypedef struct OSSL_LIB_CTX: + pass + + int OSSL_HPKE_MODE_BASE + int OSSL_HPKE_ROLE_SENDER + int OSSL_HPKE_ROLE_RECEIVER + uint16_t OSSL_HPKE_KEM_ID_X25519 + uint16_t OSSL_HPKE_KDF_ID_HKDF_SHA256 + uint16_t OSSL_HPKE_AEAD_ID_AES_GCM_256 + + OSSL_HPKE_CTX *OSSL_HPKE_CTX_new(int mode, OSSL_HPKE_SUITE suite, int role, + OSSL_LIB_CTX *libctx, const char *propq) + void OSSL_HPKE_CTX_free(OSSL_HPKE_CTX *ctx) + int OSSL_HPKE_encap(OSSL_HPKE_CTX *ctx, unsigned char *enc, size_t *enclen, + const unsigned char *pub, size_t publen, + const unsigned char *info, size_t infolen) + int OSSL_HPKE_seal(OSSL_HPKE_CTX *ctx, unsigned char *ct, size_t *ctlen, + const unsigned char *aad, size_t aadlen, + const unsigned char *pt, size_t ptlen) + int OSSL_HPKE_decap(OSSL_HPKE_CTX *ctx, const unsigned char *enc, size_t enclen, + EVP_PKEY *recippriv, const unsigned char *info, size_t infolen) + int OSSL_HPKE_open(OSSL_HPKE_CTX *ctx, unsigned char *pt, size_t *ptlen, + const unsigned char *aad, size_t aadlen, + const unsigned char *ct, size_t ctlen) + size_t OSSL_HPKE_get_public_encap_size(OSSL_HPKE_SUITE suite) + size_t OSSL_HPKE_get_ciphertext_size(OSSL_HPKE_SUITE suite, size_t clearlen) + + import struct _int = struct.Struct('>I') @@ -669,6 +729,174 @@ def blake2b_128(data): return hashlib.blake2b(data, digest_size=16).digest() +# Asymmetric primitives used for monitoring reports: Ed25519 signatures (authenticity) +# and HPKE (RFC 9180) sealing (confidentiality from the untrusted repo server). Both are +# provided by OpenSSL >= 3.2; key material is 32-byte seeds derived from the borg key. + +ED25519_SEED_SIZE = 32 +ED25519_PUBLIC_SIZE = 32 +ED25519_SIGNATURE_SIZE = 64 +X25519_SEED_SIZE = 32 +X25519_PUBLIC_SIZE = 32 + + +cdef OSSL_HPKE_SUITE _hpke_suite(): + # DHKEM(X25519, HKDF-SHA256), HKDF-SHA256, AES-256-GCM + cdef OSSL_HPKE_SUITE suite + suite.kem_id = OSSL_HPKE_KEM_ID_X25519 + suite.kdf_id = OSSL_HPKE_KDF_ID_HKDF_SHA256 + suite.aead_id = OSSL_HPKE_AEAD_ID_AES_GCM_256 + return suite + + +cdef bytes _raw_public_key(int pkey_type, bytes seed): + if len(seed) != 32: + raise ValueError("raw key seed must be 32 bytes") + cdef EVP_PKEY *pkey = EVP_PKEY_new_raw_private_key(pkey_type, NULL, PyBytes_AsString(seed), 32) + if pkey == NULL: + raise CryptoError("EVP_PKEY_new_raw_private_key failed") + cdef unsigned char pub[32] + cdef size_t publen = 32 + try: + if not EVP_PKEY_get_raw_public_key(pkey, pub, &publen): + raise CryptoError("EVP_PKEY_get_raw_public_key failed") + return PyBytes_FromStringAndSize( pub, publen) + finally: + EVP_PKEY_free(pkey) + + +def ed25519_public_from_seed(bytes seed): + """Return the 32-byte Ed25519 public key for a 32-byte secret seed.""" + return _raw_public_key(EVP_PKEY_ED25519, seed) + + +def x25519_public_from_seed(bytes seed): + """Return the 32-byte X25519 (HPKE) public key for a 32-byte secret seed.""" + return _raw_public_key(EVP_PKEY_X25519, seed) + + +def ed25519_sign(bytes seed, bytes data): + """Sign *data* with the Ed25519 secret *seed* (32 bytes), returning a 64-byte signature.""" + if len(seed) != ED25519_SEED_SIZE: + raise ValueError("ed25519 seed must be 32 bytes") + cdef EVP_PKEY *pkey = EVP_PKEY_new_raw_private_key(EVP_PKEY_ED25519, NULL, PyBytes_AsString(seed), 32) + if pkey == NULL: + raise CryptoError("EVP_PKEY_new_raw_private_key(ED25519) failed") + cdef EVP_MD_CTX *mdctx = EVP_MD_CTX_new() + cdef unsigned char sig[64] + cdef size_t siglen = 64 + try: + if mdctx == NULL: + raise CryptoError("EVP_MD_CTX_new failed") + if not EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey): + raise CryptoError("EVP_DigestSignInit failed") + if not EVP_DigestSign(mdctx, sig, &siglen, PyBytes_AsString(data), len(data)): + raise CryptoError("EVP_DigestSign failed") + return PyBytes_FromStringAndSize( sig, siglen) + finally: + if mdctx != NULL: + EVP_MD_CTX_free(mdctx) + EVP_PKEY_free(pkey) + + +def ed25519_verify(bytes public, bytes data, bytes signature): + """Verify an Ed25519 *signature* over *data* with the 32-byte *public* key. + + Returns None on success, raises IntegrityError on a bad signature. + """ + if len(public) != ED25519_PUBLIC_SIZE: + raise ValueError("ed25519 public key must be 32 bytes") + cdef EVP_PKEY *pkey = EVP_PKEY_new_raw_public_key(EVP_PKEY_ED25519, NULL, PyBytes_AsString(public), 32) + if pkey == NULL: + raise CryptoError("EVP_PKEY_new_raw_public_key(ED25519) failed") + cdef EVP_MD_CTX *mdctx = EVP_MD_CTX_new() + cdef int rc + try: + if mdctx == NULL: + raise CryptoError("EVP_MD_CTX_new failed") + if not EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey): + raise CryptoError("EVP_DigestVerifyInit failed") + rc = EVP_DigestVerify(mdctx, PyBytes_AsString(signature), len(signature), + PyBytes_AsString(data), len(data)) + if rc != 1: + raise IntegrityError("Ed25519 signature verification failed") + finally: + if mdctx != NULL: + EVP_MD_CTX_free(mdctx) + EVP_PKEY_free(pkey) + + +def hpke_seal(bytes recipient_public, bytes info, bytes aad, bytes plaintext): + """HPKE-seal *plaintext* to the recipient's 32-byte X25519 *public* key. + + Returns enc || ciphertext (the encapsulated key prepended to the AEAD ciphertext). + """ + if len(recipient_public) != X25519_PUBLIC_SIZE: + raise ValueError("recipient public key must be 32 bytes") + cdef OSSL_HPKE_SUITE suite = _hpke_suite() + cdef OSSL_HPKE_CTX *ctx = OSSL_HPKE_CTX_new(OSSL_HPKE_MODE_BASE, suite, OSSL_HPKE_ROLE_SENDER, NULL, NULL) + cdef size_t enclen = OSSL_HPKE_get_public_encap_size(suite) + cdef size_t ctlen = OSSL_HPKE_get_ciphertext_size(suite, len(plaintext)) + cdef unsigned char *enc = malloc(enclen) + cdef unsigned char *ct = malloc(ctlen) + try: + if ctx == NULL or enc == NULL or ct == NULL: + raise CryptoError("HPKE sender setup failed") + if not OSSL_HPKE_encap(ctx, enc, &enclen, + PyBytes_AsString(recipient_public), 32, + PyBytes_AsString(info), len(info)): + raise CryptoError("OSSL_HPKE_encap failed") + if not OSSL_HPKE_seal(ctx, ct, &ctlen, + PyBytes_AsString(aad), len(aad), + PyBytes_AsString(plaintext), len(plaintext)): + raise CryptoError("OSSL_HPKE_seal failed") + return PyBytes_FromStringAndSize( enc, enclen) + PyBytes_FromStringAndSize( ct, ctlen) + finally: + if enc != NULL: + free(enc) + if ct != NULL: + free(ct) + if ctx != NULL: + OSSL_HPKE_CTX_free(ctx) + + +def hpke_open(bytes recipient_secret, bytes info, bytes aad, bytes blob): + """HPKE-open a *blob* (enc || ciphertext) with the recipient's 32-byte X25519 secret. + + Returns the plaintext, raises IntegrityError if opening/authentication fails. + """ + if len(recipient_secret) != X25519_SEED_SIZE: + raise ValueError("recipient secret key must be 32 bytes") + cdef OSSL_HPKE_SUITE suite = _hpke_suite() + cdef size_t enclen = OSSL_HPKE_get_public_encap_size(suite) + if len(blob) < enclen: + raise IntegrityError("HPKE blob too short") + cdef bytes enc = blob[:enclen] + cdef bytes ct = blob[enclen:] + cdef EVP_PKEY *recippriv = EVP_PKEY_new_raw_private_key(EVP_PKEY_X25519, NULL, PyBytes_AsString(recipient_secret), 32) + cdef OSSL_HPKE_CTX *ctx = OSSL_HPKE_CTX_new(OSSL_HPKE_MODE_BASE, suite, OSSL_HPKE_ROLE_RECEIVER, NULL, NULL) + cdef size_t ptlen = len(ct) + 1 + cdef unsigned char *pt = malloc(ptlen) + try: + if recippriv == NULL or ctx == NULL or pt == NULL: + raise CryptoError("HPKE receiver setup failed") + if not OSSL_HPKE_decap(ctx, PyBytes_AsString(enc), len(enc), + recippriv, PyBytes_AsString(info), len(info)): + raise IntegrityError("OSSL_HPKE_decap failed") + if not OSSL_HPKE_open(ctx, pt, &ptlen, + PyBytes_AsString(aad), len(aad), + PyBytes_AsString(ct), len(ct)): + raise IntegrityError("OSSL_HPKE_open failed") + return PyBytes_FromStringAndSize( pt, ptlen) + finally: + if pt != NULL: + free(pt) + if ctx != NULL: + OSSL_HPKE_CTX_free(ctx) + if recippriv != NULL: + EVP_PKEY_free(recippriv) + + cdef class CSPRNG: """ Cryptographically Secure Pseudo-Random Number Generator based on AES-CTR mode. diff --git a/src/borg/crypto/monitoring.py b/src/borg/crypto/monitoring.py new file mode 100644 index 0000000000..01441fc4f8 --- /dev/null +++ b/src/borg/crypto/monitoring.py @@ -0,0 +1,111 @@ +"""Crypto for monitoring reports published into the (untrusted) repository. + +A backup client publishes a small state report into the repo's ``monitoring/`` +namespace after each relevant operation. A monitoring system pulls and verifies it +from the same untrusted server, needing no repo passphrase and no trust in the server: + +- **Authenticity** comes from an Ed25519 signature. The client signs; the monitor only + ever holds the public verify key, so neither the server nor the monitor can forge a + report. +- **Confidentiality from the server** comes from HPKE-sealing the signed payload to the + monitor's X25519 public key, so the server sees ciphertext only. + +All key material is derived deterministically from the existing borg key using borg's +own ``derive_key()`` (the same one-step KDF used for session keys) with fixed monitoring +labels and no random salt. Nothing extra is generated, stored, or rotated. + +For ``--encryption none`` repositories there is no borg key to derive from, so reports +are published as plain JSON, unsigned and unencrypted - consistent with such a repo +being unsafe anyway. The monitor must treat those as untrusted. +""" + +from binascii import hexlify, unhexlify + +from . import low_level +from .key import PlaintextKey + +# derive_key domains (labels). Fixed and distinct so the two seeds are independent. +SIGN_DOMAIN = b"borg-monitoring-sign" +SEAL_DOMAIN = b"borg-monitoring-seal" + +# HPKE info string, bound into the sealed context (domain separation / versioning). +HPKE_INFO = b"borg-monitoring-report-v1" + +# BORG_MONITORING_KEY wire format: ":". Bump on incompatible changes. +MONITOR_KEY_PREFIX = "v1:" + + +def is_signed_repo(key): + """True if this repo has a real borg key, so reports can be signed and sealed.""" + return not isinstance(key, PlaintextKey) + + +def _derive_seed(key, domain, size): + # Same one-step KDF as session keys (sha256(crypt_key + salt + domain)), but with a + # fixed label and NO random salt, so the result is deterministic. Derived from + # crypt_key (not id_key, which related repos share). + return key.derive_key(salt=b"", domain=domain, size=size) + + +def client_material(key): + """Client half: (ed25519_sign_seed, hpke_recipient_public). + + Used by the publishing side to sign with the Ed25519 secret seed and seal + to the monitor's HPKE public key. + """ + sign_seed = _derive_seed(key, SIGN_DOMAIN, low_level.ED25519_SEED_SIZE) + seal_seed = _derive_seed(key, SEAL_DOMAIN, low_level.X25519_SEED_SIZE) + hpke_public = low_level.x25519_public_from_seed(seal_seed) + return sign_seed, hpke_public + + +def monitor_material(key): + """Monitor half: (ed25519_verify_public, hpke_recipient_secret). + + This is everything the monitoring system needs to verify and decrypt and + nothing more: it cannot derive the signing secret or the borg key from it. + """ + sign_seed = _derive_seed(key, SIGN_DOMAIN, low_level.ED25519_SEED_SIZE) + seal_seed = _derive_seed(key, SEAL_DOMAIN, low_level.X25519_SEED_SIZE) + ed_public = low_level.ed25519_public_from_seed(sign_seed) + return ed_public, seal_seed + + +def export_monitor_key(key): + """Return the env-safe BORG_MONITORING_KEY string (the monitor half).""" + ed_public, hpke_secret = monitor_material(key) + return MONITOR_KEY_PREFIX + hexlify(ed_public + hpke_secret).decode("ascii") + + +def parse_monitor_key(text): + """Parse a BORG_MONITORING_KEY string into (ed25519_verify_public, hpke_secret).""" + text = text.strip() + if not text.startswith(MONITOR_KEY_PREFIX): + raise ValueError("BORG_MONITORING_KEY: unsupported format/version") + raw = unhexlify(text[len(MONITOR_KEY_PREFIX) :]) + if len(raw) != low_level.ED25519_PUBLIC_SIZE + low_level.X25519_SEED_SIZE: + raise ValueError("BORG_MONITORING_KEY: wrong length") + return raw[: low_level.ED25519_PUBLIC_SIZE], raw[low_level.ED25519_PUBLIC_SIZE :] + + +def seal_report(key, payload, aad): + """Sign *payload* (bytes) with the client's Ed25519 secret, then HPKE-seal it. + + *aad* (bytes, e.g. the repo id) is bound into both the HPKE seal context, so a sealed + report cannot be transplanted to a different repo. Returns the sealed envelope bytes. + """ + sign_seed, hpke_public = client_material(key) + signature = low_level.ed25519_sign(sign_seed, payload) + return low_level.hpke_seal(hpke_public, HPKE_INFO, aad, signature + payload) + + +def open_report(ed_public, hpke_secret, blob, aad): + """HPKE-open *blob*, verify the Ed25519 signature, and return the *payload* bytes. + + Raises low_level.IntegrityError if decryption or signature verification fails. + """ + signed = low_level.hpke_open(hpke_secret, HPKE_INFO, aad, blob) + siglen = low_level.ED25519_SIGNATURE_SIZE + signature, payload = signed[:siglen], signed[siglen:] + low_level.ed25519_verify(ed_public, payload, signature) + return payload diff --git a/src/borg/monitoring.py b/src/borg/monitoring.py new file mode 100644 index 0000000000..6b28784bf9 --- /dev/null +++ b/src/borg/monitoring.py @@ -0,0 +1,210 @@ +"""Build, publish and read monitoring reports (see crypto/monitoring.py for the crypto). + +Each backup-side command (create, prune, ...) appends one report object to the repo's +``monitoring/`` namespace; a monitoring system reads them back and verifies them. The +on-disk object is:: + + byte 0: format version + byte 1: body type (0 = plaintext JSON, 1 = sealed: HPKE(ed25519-sign || JSON)) + rest: body + +The HPKE seal is bound (via aad) to the repository id, which both sides obtain from the +repository they are talking to - it is never trusted from inside the ciphertext. + +Objects are append-only and named by publish time, so reports for different archive +series never overwrite each other (e.g. a failed home backup is not masked by a later +successful system backup). The namespace is bounded by ``borg monitor --keep=N``, which +deletes all but the N newest objects. +""" + +import json +import logging +import os +import time +from binascii import hexlify +from getpass import getuser + +from borgstore.store import ItemInfo +from borgstore.store import ObjectNotFound as StoreObjectNotFound + +from . import __version__ +from . import platform +from .constants import EXIT_SUCCESS, EXIT_WARNING, EXIT_WARNING_BASE, EXIT_SIGNAL_BASE +from .crypto import monitoring as mon_crypto +from .helpers import bin_to_hex, get_ec +from .helpers.time import archive_ts_now + +logger = logging.getLogger(__name__) + +STORE_NAMESPACE = "monitoring" + +# Default number of newest report objects to keep when running "borg monitor". +DEFAULT_KEEP = 500 + + +def _new_object_name(): + """A unique, chronologically sortable object name. + + Microseconds since the epoch, zero-padded to a fixed width so lexical sort equals + chronological order, plus a random suffix so same-microsecond or concurrent publishes + never collide (and never overwrite an existing report). + """ + us = int(time.time() * 1_000_000) + return f"{us:020d}.{hexlify(os.urandom(4)).decode('ascii')}" + + +FORMAT_VERSION = 1 +BODY_PLAIN = 0 +BODY_SEALED = 1 + + +def status_from_rc(rc): + """Map a borg return code to a coarse status string. + + See constants.py: 0 = success, 1 = generic warning, 100..127 = specific warnings, + everything else (generic/specific errors, signals) = error. + """ + if rc == EXIT_SUCCESS: + return "success" + if rc == EXIT_WARNING or EXIT_WARNING_BASE <= rc < EXIT_SIGNAL_BASE: + return "warning" + return "error" + + +def build_report( + *, command, repo_id, time, rc, hostname=None, username=None, archive=None, archive_id=None, stats=None +): + """Assemble the report dict. *repo_id*/*archive_id* are hex strings, *time* is ISO.""" + report = { + "borg_version": __version__, + "repo_id": repo_id, + "command": command, + "time": time, + "status": status_from_rc(rc), + "rc": rc, + } + if hostname is not None: + report["hostname"] = hostname + if username is not None: + report["username"] = username + if archive is not None: + report["archive"] = archive + if archive_id is not None: + report["archive_id"] = archive_id + if stats is not None: + report["stats"] = stats + return report + + +def serialize(key, repo_id_bin, report): + """Serialize *report* into the on-disk object bytes, sealing it if the repo is encrypted.""" + payload = json.dumps(report, sort_keys=True).encode("utf-8") + if mon_crypto.is_signed_repo(key): + body = mon_crypto.seal_report(key, payload, repo_id_bin) + body_type = BODY_SEALED + else: + body = payload + body_type = BODY_PLAIN + return bytes([FORMAT_VERSION, body_type]) + body + + +def deserialize(monitor_key, repo_id_bin, data): + """Return (report_dict, trusted: bool). + + *monitor_key* is the parsed (ed25519_public, hpke_secret) tuple, or None. A sealed + report is verified+decrypted (trusted=True); a plaintext report is returned as-is + (trusted=False). Raises ValueError/IntegrityError on malformed or unverifiable data. + """ + if len(data) < 2 or data[0] != FORMAT_VERSION: + raise ValueError("monitoring report: unsupported format version") + body_type, body = data[1], data[2:] + if body_type == BODY_SEALED: + if monitor_key is None: + raise ValueError("monitoring report is sealed but no BORG_MONITORING_KEY was given") + ed_public, hpke_secret = monitor_key + payload = mon_crypto.open_report(ed_public, hpke_secret, body, repo_id_bin) + trusted = True + elif body_type == BODY_PLAIN: + payload = body + trusted = False + else: + raise ValueError("monitoring report: unknown body type") + return json.loads(payload.decode("utf-8")), trusted + + +def publish(repository, key, report): + """Append *report* to the repository as a new object. Best-effort: never raise out.""" + try: + data = serialize(key, repository.id, report) + repository.store_store(f"{STORE_NAMESPACE}/{_new_object_name()}", data) + except Exception as exc: + logger.warning("Could not publish monitoring report: %s", exc) + + +def publish_command_report( + repository, key, command, *, hostname=None, username=None, archive=None, archive_id=None, stats=None +): + """Build and publish a report for a finished command. + + Captures the best-known return code at call time (the true process rc is only final + after the store is closed; see borg/monitoring.py). Call this as the last action while + the store is still open. *archive_id* is binary; it is hex-encoded for the report. + *hostname*/*username* default to the local host and user (e.g. for repo-wide commands + like prune); callers with an archive should pass the archive's own host/user. + """ + report = build_report( + command=command, + repo_id=bin_to_hex(repository.id), + time=archive_ts_now().isoformat(timespec="microseconds"), + rc=get_ec(), + hostname=hostname if hostname is not None else platform.hostname, + username=username if username is not None else getuser(), + archive=archive, + archive_id=bin_to_hex(archive_id) if archive_id is not None else None, + stats=stats, + ) + publish(repository, key, report) + + +def list_names(repository): + """Return all monitoring object names, oldest first (names sort chronologically).""" + names = [ItemInfo(*info).name for info in repository.store_list(STORE_NAMESPACE)] + names.sort() + return names + + +def iter_reports(repository, monitor_key): + """Yield (report, trusted) for every stored report, oldest first. + + Each report is verified and decrypted; an unverifiable one raises (it is not silently + skipped) so tampering surfaces. + """ + for name in list_names(repository): + try: + data = repository.store_load(f"{STORE_NAMESPACE}/{name}") + except StoreObjectNotFound: + continue # raced with a concurrent --keep cleanup + yield deserialize(monitor_key, repository.id, data) + + +def prune_reports(repository, keep): + """Delete all but the *keep* newest report objects. Best-effort; returns #deleted. + + Needs delete permission on the monitoring namespace; on a permission error (e.g. a + read-only monitoring host) it warns and stops rather than failing the command. + """ + if keep is None or keep <= 0: + return 0 # 0 (or negative) disables cleanup + names = list_names(repository) + to_delete = names[:-keep] + deleted = 0 + for name in to_delete: + try: + repository.store_delete(f"{STORE_NAMESPACE}/{name}") + deleted += 1 + except StoreObjectNotFound: + pass # already gone (concurrent cleanup) + except Exception as exc: + logger.warning("Could not delete old monitoring report %s: %s", name, exc) + break # likely missing delete permission; do not hammer the server + return deleted diff --git a/src/borg/repository.py b/src/borg/repository.py index f6c10f731e..fccfffa021 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -53,6 +53,7 @@ def borg_permissions(permissions): "index": "lrwWD", # WD for index/ (merge/compaction of incremental indexes) "keys": "lr", "locks": "lrwD", # borg needs to create/delete a shared lock here + "monitoring": "lrw", # append new report objects (cleanup is done by borg monitor) "packs": "lrw", } case "write-only": # mostly no reading @@ -64,10 +65,13 @@ def borg_permissions(permissions): "index": "lrwWD", # read allowed so that borg create can check chunk presence for deduplication "keys": "lr", "locks": "lrwD", # borg needs to create/delete a shared lock here + "monitoring": "lw", # append new report objects (cleanup is done by borg monitor) "packs": "lw", # no r! } case "read-only": # mostly r/o - return {"": "lr", "locks": "lrwD"} + # "monitoring": lrD lets a restricted monitoring host run "borg monitor --keep" + # (read all reports, delete old ones) without any other write access. + return {"": "lr", "locks": "lrwD", "monitoring": "lrD"} case _: raise Error( f"Invalid BORG_REPO_PERMISSIONS value: {permissions}, should be one of: " @@ -305,6 +309,7 @@ def __init__( "index/": {"levels": [0]}, "keys/": {"levels": [0]}, "locks/": {"levels": [0]}, + "monitoring/": {"levels": [0]}, "packs/": {"levels": [1]}, } # Get permissions from parameter or environment variable diff --git a/src/borg/testsuite/archiver/monitor_cmd_test.py b/src/borg/testsuite/archiver/monitor_cmd_test.py new file mode 100644 index 0000000000..6ca86dd3ee --- /dev/null +++ b/src/borg/testsuite/archiver/monitor_cmd_test.py @@ -0,0 +1,206 @@ +import json +import os + +from ...constants import * # NOQA +from . import cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION + +pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local") # NOQA + + +def _monitoring_key(archiver): + """Derive BORG_MONITORING_KEY via `borg monitor --key` (needs the borg key).""" + output = cmd(archiver, "monitor", "--key") + keys = [line.strip() for line in output.splitlines() if line.strip().startswith("v1:")] + assert len(keys) == 1, output + return keys[0] + + +def _entries(archiver, *extra): + """Run `borg monitor --json` and return {archive-or-command: entry}.""" + data = json.loads(cmd(archiver, "monitor", "--json", *extra)) + return {(e["archive"] or e["command"]): e for e in data["entries"]} + + +def _monitoring_object_count(archiver): + return len(os.listdir(os.path.join(archiver.repository_path, "monitoring"))) + + +def test_create_publishes_report_and_monitor_reads_it(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "archive1", "input") + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + + output = cmd(archiver, "monitor") + assert "status: success" in output + assert "trusted: True" in output + assert "archive1" in output + + entries = _entries(archiver) + assert set(entries) == {"archive1"} + e = entries["archive1"] + assert e["trusted"] is True and e["stale"] is False + assert e["report"]["command"] == "create" + assert e["report"]["status"] == "success" + assert "archive_id" in e["report"] + # host/user metadata is recorded and surfaced on the entry + assert e["hostname"] and e["hostname"] == e["report"]["hostname"] + assert e["username"] and e["username"] == e["report"]["username"] + + +def test_multiple_series_are_not_masked(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + # two independent archive series backed up by the same command + cmd(archiver, "create", "backup-home", "input") + cmd(archiver, "create", "backup-system", "input") + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + + # both series are reported independently - the later one does not overwrite the earlier + entries = _entries(archiver) + assert set(entries) == {"backup-home", "backup-system"} + + # --name restricts to a single series + data = json.loads(cmd(archiver, "monitor", "--name", "backup-home", "--json")) + assert [e["archive"] for e in data["entries"]] == ["backup-home"] + + +def test_same_series_from_different_hosts_kept_separate(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + # same archive series name, but backed up from two different hosts to the same repo + cmd(archiver, "create", "--hostname", "host-a", "shared", "input") + cmd(archiver, "create", "--hostname", "host-b", "shared", "input") + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + + data = json.loads(cmd(archiver, "monitor", "--json")) + assert sorted(e["hostname"] for e in data["entries"]) == ["host-a", "host-b"] + assert all(e["archive"] == "shared" for e in data["entries"]) + + # --host narrows to a single host + data = json.loads(cmd(archiver, "monitor", "--host", "host-a", "--json")) + assert [e["hostname"] for e in data["entries"]] == ["host-a"] + + # an unknown host matches nothing -> dead man's switch fires + out = cmd(archiver, "monitor", "--host", "nope", exit_code=EXIT_ERROR) + assert "No monitoring report" in out + + +def test_prune_publishes_its_own_report(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + # one series with two archives, then prune down to one + cmd(archiver, "create", "series", "input") + cmd(archiver, "create", "series", "input") + cmd(archiver, "prune", "--keep-last", "1", "series") + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + + entries = _entries(archiver) + assert set(entries) == {"series", "prune"} + assert entries["series"]["report"]["command"] == "create" + prune = entries["prune"]["report"] + assert prune["command"] == "prune" + assert prune["stats"]["archives_pruned"] == 1 + assert prune["stats"]["archives_kept"] == 1 + + +def test_delete_and_undelete_publish_reports(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "a1", "input") + cmd(archiver, "create", "a2", "input") + cmd(archiver, "delete", "a2") + cmd(archiver, "undelete", "a2") + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + + entries = _entries(archiver) + assert {"delete", "undelete"} <= set(entries) + assert entries["delete"]["report"]["stats"]["archives_deleted"] == 1 + assert entries["undelete"]["report"]["stats"]["archives_undeleted"] == 1 + + +def test_transfer_publishes_report(archivers, request, monkeypatch): + from .transfer_cmd_test import setup_repos + + archiver = request.getfixturevalue(archivers) + with setup_repos(archiver, monkeypatch) as other_repo1: + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "create", "arch1", "input") + cmd(archiver, "create", "arch2", "input") + cmd(archiver, "transfer", other_repo1) + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + + entries = _entries(archiver) + assert "transfer" in entries + stats = entries["transfer"]["report"]["stats"] + assert stats["archives_transferred"] == 2 + assert stats["archives_considered"] == 2 + + +def test_keep_evicts_old_objects(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + for _ in range(4): + cmd(archiver, "create", "series", "input") + assert _monitoring_object_count(archiver) == 4 + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + # reading with --keep deletes all but the N newest objects + cmd(archiver, "monitor", "--keep", "2") + assert _monitoring_object_count(archiver) == 2 + # --keep 0 disables cleanup + cmd(archiver, "monitor", "--keep", "0") + assert _monitoring_object_count(archiver) == 2 + + +def test_monitor_stale_report_alerts(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "archive1", "input") + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + # a zero freshness window makes any report stale -> error exit code (dead man's switch) + output = cmd(archiver, "monitor", "--max-age", "0", exit_code=EXIT_ERROR) + assert "STALE" in output + + +def test_monitor_no_report(archivers, request, monkeypatch): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + monkeypatch.setenv("BORG_MONITORING_KEY", _monitoring_key(archiver)) + output = cmd(archiver, "monitor", exit_code=EXIT_ERROR) + assert "No monitoring report" in output + + +def test_monitor_without_key_errors(archivers, request): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "archive1", "input") + # sealed report but no BORG_MONITORING_KEY -> clean Error (use fork so it maps to a rc) + output = cmd(archiver, "monitor", fork=True, exit_code=EXIT_ERROR) + assert "BORG_MONITORING_KEY" in output + + +def test_monitor_unencrypted_repo_is_untrusted(archivers, request): + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "file1", contents=b"some data") + cmd(archiver, "repo-create", "--encryption=none") + cmd(archiver, "create", "archive1", "input") + # no key needed; report is plaintext and flagged untrusted -> warning exit code + output = cmd(archiver, "monitor", exit_code=EXIT_WARNING) + assert "trusted: False" in output + # there is no monitoring key to export for an unencrypted repo + out = cmd(archiver, "monitor", "--key", fork=True, exit_code=EXIT_ERROR) + assert "unencrypted" in out + + +def test_monitor_key_export_is_deterministic(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + assert _monitoring_key(archiver) == _monitoring_key(archiver) diff --git a/src/borg/testsuite/monitoring_test.py b/src/borg/testsuite/monitoring_test.py new file mode 100644 index 0000000000..6094f2f096 --- /dev/null +++ b/src/borg/testsuite/monitoring_test.py @@ -0,0 +1,212 @@ +import os + +import pytest + +from ..constants import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, EXIT_SIGNAL_BASE +from ..crypto import low_level +from ..crypto.key import AESOCBKey, PlaintextKey +from ..crypto import monitoring as mc +from .. import monitoring as m + + +def make_key(): + """A minimal AEAD key with a real random crypt_key (enough for derive_key).""" + key = AESOCBKey.__new__(AESOCBKey) + key.crypt_key = os.urandom(32) + key.id_key = os.urandom(32) + return key + + +# --- low-level crypto primitives ------------------------------------------------------- + + +def test_ed25519_sign_verify_roundtrip(): + seed = os.urandom(32) + pub = low_level.ed25519_public_from_seed(seed) + assert len(pub) == 32 + sig = low_level.ed25519_sign(seed, b"hello") + assert len(sig) == 64 + low_level.ed25519_verify(pub, b"hello", sig) # must not raise + + +def test_ed25519_rejects_tampered_message(): + seed = os.urandom(32) + pub = low_level.ed25519_public_from_seed(seed) + sig = low_level.ed25519_sign(seed, b"hello") + with pytest.raises(low_level.IntegrityError): + low_level.ed25519_verify(pub, b"hello!", sig) + + +def test_ed25519_rejects_wrong_key(): + seed = os.urandom(32) + sig = low_level.ed25519_sign(seed, b"hello") + other_pub = low_level.ed25519_public_from_seed(os.urandom(32)) + with pytest.raises(low_level.IntegrityError): + low_level.ed25519_verify(other_pub, b"hello", sig) + + +def test_hpke_seal_open_roundtrip(): + rseed = os.urandom(32) + rpub = low_level.x25519_public_from_seed(rseed) + blob = low_level.hpke_seal(rpub, b"info", b"aad", b"secret payload") + assert low_level.hpke_open(rseed, b"info", b"aad", blob) == b"secret payload" + + +def test_hpke_rejects_wrong_recipient(): + rpub = low_level.x25519_public_from_seed(os.urandom(32)) + blob = low_level.hpke_seal(rpub, b"info", b"aad", b"secret") + with pytest.raises(low_level.IntegrityError): + low_level.hpke_open(os.urandom(32), b"info", b"aad", blob) + + +def test_hpke_rejects_wrong_aad(): + rseed = os.urandom(32) + rpub = low_level.x25519_public_from_seed(rseed) + blob = low_level.hpke_seal(rpub, b"info", b"aad", b"secret") + with pytest.raises(low_level.IntegrityError): + low_level.hpke_open(rseed, b"info", b"other-aad", blob) + + +# --- key derivation -------------------------------------------------------------------- + + +def test_derivation_is_deterministic(): + key = make_key() + assert mc.client_material(key) == mc.client_material(key) + assert mc.monitor_material(key) == mc.monitor_material(key) + + +def test_client_and_monitor_halves_are_consistent(): + key = make_key() + sign_seed, hpke_public = mc.client_material(key) + ed_public, hpke_secret = mc.monitor_material(key) + assert ed_public == low_level.ed25519_public_from_seed(sign_seed) + assert hpke_public == low_level.x25519_public_from_seed(hpke_secret) + + +def test_labels_yield_independent_keys(): + key = make_key() + sign_seed, _ = mc.client_material(key) + _, hpke_secret = mc.monitor_material(key) + assert sign_seed != hpke_secret + + +def test_monitor_half_does_not_contain_signing_secret(): + key = make_key() + sign_seed, _ = mc.client_material(key) + ed_public, hpke_secret = mc.monitor_material(key) + assert sign_seed not in (ed_public, hpke_secret) + + +def test_derivation_is_per_key_unique(): + assert mc.client_material(make_key()) != mc.client_material(make_key()) + + +def test_export_parse_monitor_key_roundtrip(): + key = make_key() + text = mc.export_monitor_key(key) + assert text.startswith(mc.MONITOR_KEY_PREFIX) + assert mc.parse_monitor_key(text) == mc.monitor_material(key) + + +def test_parse_monitor_key_rejects_bad_input(): + with pytest.raises(ValueError): + mc.parse_monitor_key("nope") + with pytest.raises(ValueError): + mc.parse_monitor_key("v1:00") + + +def test_plaintext_repo_is_not_signed(): + assert mc.is_signed_repo(make_key()) is True + assert mc.is_signed_repo(PlaintextKey.__new__(PlaintextKey)) is False + + +# --- report build / serialize / deserialize ------------------------------------------- + + +def sample_report(repo_id_hex): + return m.build_report( + command="create", + repo_id=repo_id_hex, + time="2026-06-17T11:59:58.123456+00:00", + rc=EXIT_SUCCESS, + archive="host-2026-06-17", + archive_id="aa" * 32, + stats={"original_size": 10485760, "nfiles": 1234}, + ) + + +def test_status_from_rc(): + assert m.status_from_rc(EXIT_SUCCESS) == "success" + assert m.status_from_rc(EXIT_WARNING) == "warning" + assert m.status_from_rc(100) == "warning" # specific warning range + assert m.status_from_rc(EXIT_ERROR) == "error" + assert m.status_from_rc(3) == "error" # specific error range + assert m.status_from_rc(EXIT_SIGNAL_BASE + 2) == "error" + + +def test_build_report_schema(): + report = sample_report("bb" * 32) + assert report["command"] == "create" + assert report["status"] == "success" + assert report["archive_id"] == "aa" * 32 + assert "borg_version" in report and "time" in report + + +def test_sealed_report_roundtrip_and_trusted(): + key = make_key() + repo_id = os.urandom(32) + report = sample_report(repo_id.hex()) + data = m.serialize(key, repo_id, report) + assert data[0] == m.FORMAT_VERSION and data[1] == m.BODY_SEALED + monitor_key = mc.parse_monitor_key(mc.export_monitor_key(key)) + got, trusted = m.deserialize(monitor_key, repo_id, data) + assert got == report and trusted is True + + +def test_sealed_report_rejects_wrong_repo_id(): + key = make_key() + repo_id = os.urandom(32) + data = m.serialize(key, repo_id, sample_report(repo_id.hex())) + monitor_key = mc.parse_monitor_key(mc.export_monitor_key(key)) + with pytest.raises(low_level.IntegrityError): + m.deserialize(monitor_key, os.urandom(32), data) + + +def test_sealed_report_rejects_tamper(): + key = make_key() + repo_id = os.urandom(32) + data = bytearray(m.serialize(key, repo_id, sample_report(repo_id.hex()))) + data[-1] ^= 1 + monitor_key = mc.parse_monitor_key(mc.export_monitor_key(key)) + with pytest.raises(low_level.IntegrityError): + m.deserialize(monitor_key, repo_id, bytes(data)) + + +def test_sealed_report_rejects_wrong_monitor_key(): + key = make_key() + repo_id = os.urandom(32) + data = m.serialize(key, repo_id, sample_report(repo_id.hex())) + wrong = mc.parse_monitor_key(mc.export_monitor_key(make_key())) + with pytest.raises(low_level.IntegrityError): + m.deserialize(wrong, repo_id, data) + + +def test_sealed_report_requires_key(): + key = make_key() + repo_id = os.urandom(32) + data = m.serialize(key, repo_id, sample_report(repo_id.hex())) + with pytest.raises(ValueError): + m.deserialize(None, repo_id, data) + + +def test_plaintext_report_roundtrip_is_untrusted(): + key = PlaintextKey.__new__(PlaintextKey) + key.crypt_key = b"" + key.id_key = b"" + repo_id = os.urandom(32) + report = sample_report(repo_id.hex()) + data = m.serialize(key, repo_id, report) + assert data[1] == m.BODY_PLAIN + got, trusted = m.deserialize(None, repo_id, data) + assert got == report and trusted is False