diff --git a/src/borg/archive.py b/src/borg/archive.py index 271118ac86..55dcaed62b 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -32,7 +32,7 @@ from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError from .helpers import HardLinkManager from .helpers import ChunkIteratorFileWrapper, open_item -from .helpers import Error, IntegrityError, set_ec +from .helpers import Error, IntegrityError, set_ec, sig_int from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns from .helpers import parse_timestamp, archive_ts_now, CompressionSpec from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize @@ -1929,12 +1929,28 @@ def check( rebuild_manifest = True if rebuild_manifest: self.manifest = self.rebuild_manifest() - if find_lost_archives: + # On Ctrl-C, skip any scan not yet started; a scan already running stops at its own boundary. + if find_lost_archives and not sig_int: self.rebuild_archives_directory() - self.rebuild_archives( - match=match, first=first, last=last, sort_by=sort_by, older=older, oldest=oldest, newer=newer, newest=newest - ) + if not sig_int: + self.rebuild_archives( + match=match, + first=first, + last=last, + sort_by=sort_by, + older=older, + oldest=oldest, + newer=newer, + newest=newest, + ) + # finish() writes the manifest and a consistent chunk index; run it on Ctrl-C too (#9850). self.finish() + if sig_int: + if self.error_found: + logger.error("Archive consistency check interrupted, problems found so far.") + else: + logger.info("Archive consistency check interrupted, no problems found so far.") + raise Error("Got Ctrl-C / SIGINT.") if self.error_found: logger.error("Archive consistency check complete, problems found.") else: @@ -1982,12 +1998,16 @@ def verify_data(self): logger.info("Starting cryptographic data integrity verification...") chunks_count = len(self.chunks) errors = 0 + verified = 0 # chunks actually verified defect_chunks = [] pi = ProgressIndicatorPercent( total=chunks_count, msg="Verifying data %6.2f%%", step=0.01, msgid="check.verify_data" ) for chunk_id, _ in self.chunks.iteritems(): + if sig_int: + break pi.show() + verified += 1 try: encrypted_data = self.repository.get(chunk_id) except (Repository.ObjectNotFound, IntegrityErrorBase) as err: @@ -2043,11 +2063,20 @@ def verify_data(self): for defect_chunk in defect_chunks: logger.debug("chunk %s is defect.", bin_to_hex(defect_chunk)) log = logger.error if errors else logger.info - log( - "Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.", - chunks_count, - errors, - ) + if sig_int: + log( + "Interrupted cryptographic data integrity verification, " + "verified %d of %d chunks with %d integrity errors.", + verified, + chunks_count, + errors, + ) + else: + log( + "Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.", + verified, + errors, + ) def rebuild_manifest(self): """Rebuild the manifest object.""" @@ -2083,6 +2112,8 @@ def valid_archive(obj): msgid="check.rebuild_archives_directory", ) for chunk_id, _ in self.chunks.iteritems(): + if sig_int: + break pi.show() cdata = self.repository.get(chunk_id, read_data=False) # only get metadata try: @@ -2128,7 +2159,10 @@ def valid_archive(obj): logger.warning(f"Would create archives directory entry for {name} {archive_id_hex}.") pi.finish() - logger.info("Rebuilding missing archives directory entries completed.") + if sig_int: + logger.info("Rebuilding missing archives directory entries interrupted.") + else: + logger.info("Rebuilding missing archives directory entries completed.") def rebuild_archives( self, first=0, last=0, sort_by="", match=None, older=None, newer=None, oldest=None, newest=None @@ -2326,6 +2360,9 @@ def valid_item(obj): # badly damaged repo does not throw away everything it already found. try: for i, info in enumerate(archive_infos): + if sig_int: + # Break only between archives, as --repair rewrites each archive as a whole. + break pi.show(i) archive_id, archive_id_hex = info.id, bin_to_hex(info.id) try: diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 6e983a9876..db171f6729 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -3,8 +3,8 @@ from ._common import with_repository, Highlander from ..archive import ArchiveChecker from ..constants import * # NOQA -from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError -from ..helpers import relative_time_marker_validator, yes, ArchiveFormatter +from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, Error, IntegrityError +from ..helpers import relative_time_marker_validator, yes, ArchiveFormatter, sig_int from ..helpers.argparsing import ArgumentParser from ..helpers.time import archive_ts_now, calculate_relative_offset @@ -80,6 +80,8 @@ def do_check(self, args, repository): if not args.archives_only: if not repository.check(repair=args.repair, max_duration=args.max_duration, max_age=max_age): set_ec(EXIT_WARNING) + if sig_int: # repository check interrupted; skip the archive check + raise Error("Got Ctrl-C / SIGINT.") if not args.repo_only and not archive_checker.check( repository, verify_data=args.verify_data, @@ -186,6 +188,21 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser): formatted by giving a custom format using ``--format`` (see the ``borg repo-list`` description for more details about the format string). + If the ``borg check`` process receives a SIGINT signal (Ctrl-C), it stops at the + next safe boundary, leaving the repository and its chunk index in a consistent state. + The repository check stops after the current pack; ``--verify-data`` and + ``--find-lost-archives`` stop after the current chunk; a ``--repair`` archive check + stops between whole archives. A partial repository check (``--max-duration``) saves its + progress so a later partial check resumes where it stopped; a full check restarts from + the beginning. With ``--repair``, an interrupted archive check may leave some archives + already repaired and others not yet processed, so run ``borg check --repair`` again to + finish. + + During a ``--repair`` run, the archive check first rebuilds the chunk index from the + packs, and, if the key must be recovered, scans chunks for it. These phases do not yet + respond to SIGINT, so on a large repository a Ctrl-C during them may appear to have no + effect until they finish. + About repair mode +++++++++++++++++ diff --git a/src/borg/repository.py b/src/borg/repository.py index 8665bba9c3..b2ccff8157 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1057,6 +1057,10 @@ def recorded_ts(info): pack_infos.sort(key=recorded_ts) pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs") for info in pack_infos: + if sig_int: # on Ctrl-C, persist checked packs, then stop + logger.info(f"Interrupted repository check, {len(tracker)} packs checked so far.") + tracker.save() + break self._lock_refresh() pack_pi.show(increase=1) # advance for skipped packs too, so the bar tracks packs/, not work done pack_id = hex_to_bin(info.name) @@ -1110,12 +1114,14 @@ def recorded_ts(info): logger.error(f"Corrupt pack: {bin_to_hex(pack_id)}") # fail if this run found errors, or any pack is recorded corrupt. problems = objs_errors != 0 or bool(corrupt_ids) + # On Ctrl-C the check stopped early, so the summary only covers the packs seen so far. + done, so_far = ("Interrupted", " so far") if sig_int else ("Finished", "") if not problems: - logger.info(f"Finished {mode} repository check, no problems found.") + logger.info(f"{done} {mode} repository check, no problems found{so_far}.") elif repair: - logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).") + logger.error(f"{done} {mode} repository check, errors found{so_far} (repository repair not implemented).") else: - logger.error(f"Finished {mode} repository check, errors found.") + logger.error(f"{done} {mode} repository check, errors found{so_far}.") return not problems or repair def list(self, limit=None, marker=None): diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index 03d7feba6e..12abbd9aaf 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -6,11 +6,11 @@ import pytest -from ...archive import ChunkBuffer, ArchiveChecker +from ...archive import ArchiveChecker, ChunkBuffer from ...constants import * # NOQA -from ...helpers import bin_to_hex, msgpack, CommandError, IntegrityError -from ...manifest import Manifest -from ...repository import Repository +from ...helpers import bin_to_hex, msgpack, CommandError, Error, IntegrityError, sig_int +from ...manifest import Archives, Manifest +from ...repository import PackTracker, Repository from ..repository_test import fchunk, corrupt_chunk_on_disk from . import ( cmd, @@ -75,6 +75,138 @@ def test_check_usage(archivers, request): assert "archive2" in output +def test_check_soft_interrupt(archivers, request, monkeypatch): + """A mid-run Ctrl-C stops both check phases at a safe boundary (#7893): the repository check persists + its checked packs for a later partial check to resume, and the archive check runs finish() and then + raises. The check is read-only, so a normal check still passes afterwards.""" + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) # produces many packs + + # repository check: interrupt after the first pack. + with Repository(archiver.repository_path, exclusive=True) as repository: + orig_hash = repository.store.hash + pack_checks = [] + + def hash_then_interrupt(key): + result = orig_hash(key) + if key.startswith("packs/"): # count pack checks, not the index files hashed first + pack_checks.append(key) + if len(pack_checks) == 1: # one Ctrl-C after the first pack is checked + sig_int._sig_int_triggered = True + return result + + monkeypatch.setattr(repository.store, "hash", hash_then_interrupt) + try: + repository.check() + finally: + sig_int._sig_int_triggered = False + assert len(PackTracker.load(repository.store)) == 1 # the pack checked before the break persisted + + # a partial check resumes from the saved record (the one pack checked before the interrupt). + output = cmd(archiver, "check", "-v", "--repository-only", "--max-duration=600", exit_code=0) + assert "1 pack check results on record" in output + + # archive check: interrupt verify_data after 3 chunks. + with Repository(archiver.repository_path, exclusive=True) as repository: + orig_get = repository.get + get_calls = 0 + + def get_then_interrupt(*args, **kwargs): + nonlocal get_calls + get_calls += 1 + if get_calls == 3: # trip mid-loop, after 3 chunks + sig_int._sig_int_triggered = True + return orig_get(*args, **kwargs) + + monkeypatch.setattr(repository, "get", get_then_interrupt) + try: + with pytest.raises(Error, match="Got Ctrl-C"): + ArchiveChecker().check(repository, verify_data=True, sort_by="ts", format="{archive} {time} {id}") + finally: + sig_int._sig_int_triggered = False + # verify_data breaks at the chunk it interrupted on, and the skipped scans issue no more get()s. + assert get_calls == 3 + + # nothing changed, so a normal check passes. + cmd(archiver, "check", exit_code=0) + + +def test_check_repair_soft_interrupt(archivers, request, monkeypatch): + """A Ctrl-C after the first archive of a --repair archive check stops at the archive boundary, runs + finish() (dropping the chunk index, writing the manifest), then raises. No archive is lost, and a + second --repair finishes the job so a following check reports the repository consistent.""" + archiver = request.getfixturevalue(archivers) + check_cmd_setup(archiver) # two archives + + orig_create = Archives.create + + def create_then_interrupt(self, *args, **kwargs): + orig_create(self, *args, **kwargs) + sig_int._sig_int_triggered = True # one Ctrl-C after the first archive was rebuilt + + monkeypatch.setattr(Archives, "create", create_then_interrupt) + try: + with Repository(archiver.repository_path, exclusive=True) as repository: + with pytest.raises(Error, match="Got Ctrl-C"): + ArchiveChecker().check(repository, repair=True, sort_by="ts", format="{archive} {time} {id}") + finally: + sig_int._sig_int_triggered = False # reset the global flag for the following tests + # restore the real method; monkeypatch.undo() would also drop the autouse env (BORG_TESTONLY_WEAKEN_KDF). + monkeypatch.setattr(Archives, "create", orig_create) + + # both archives survive the interrupt between archives. + output = cmd(archiver, "repo-list", exit_code=0) + assert "archive1" in output + assert "archive2" in output + + # a second --repair finishes the job; a plain check then finds no problems. + cmd(archiver, "check", "--repair", exit_code=0) + cmd(archiver, "check", exit_code=0) + + +def test_check_interrupt_skips_archive_check(archivers, request, monkeypatch): + """A Ctrl-C during the repository check makes a full `borg check` skip the archive check. do_check + raises at the sig_int guard, which sits before the archive_checker.check() call, so the raise itself + is the skip. Exercises the do_check path (the other soft-interrupt tests call check() directly).""" + archiver = request.getfixturevalue(archivers) + if archiver.EXE: # a class-level monkeypatch cannot reach the borg.exe subprocess + pytest.skip("in-process store patch does not apply to the binary") + check_cmd_setup(archiver) # produces many packs + + from borgstore.store import Store + + orig_hash = Store.hash + pack_checks = [] + + def hash_then_interrupt(self, key): + result = orig_hash(self, key) + if key.startswith("packs/"): # count pack checks, not the index files hashed first + pack_checks.append(key) + if len(pack_checks) == 1: # one Ctrl-C after the first pack is checked + sig_int._sig_int_triggered = True + return result + + # spy on the archive check: "Got Ctrl-C" is also raised inside ArchiveChecker.check(), so matching the + # message alone would not prove the skip. Recording that check() never runs is the load-bearing assertion. + orig_check = ArchiveChecker.check + archive_check_ran = False + + def spy_check(self, *args, **kwargs): + nonlocal archive_check_ran + archive_check_ran = True + return orig_check(self, *args, **kwargs) + + monkeypatch.setattr(Store, "hash", hash_then_interrupt) + monkeypatch.setattr(ArchiveChecker, "check", spy_check) + try: + # exec_cmd calls Archiver.run() directly; only main() maps Error to an exit code, so it propagates. + with pytest.raises(Error, match="Got Ctrl-C"): + cmd(archiver, "check", "-v") + finally: + sig_int._sig_int_triggered = False + assert archive_check_ran is False # do_check raised at the sig_int guard, before archive_checker.check() + + def test_check_max_age(archivers, request): archiver = request.getfixturevalue(archivers) check_cmd_setup(archiver)