From 750805b21812e3fabc5b218ce64b7087d90b1d4c Mon Sep 17 00:00:00 2001 From: Mrityunjay Raj Date: Thu, 6 Aug 2026 06:34:49 +0530 Subject: [PATCH] check --repair: rebuild a corrupt repository index from the packs, #10026 Verify each pack's sha256 and rebuild the chunks index from the intact packs' object headers, then persist it. Chunks that exist only in corrupt packs are dropped; salvaging them is not implemented yet. --- src/borg/cache.py | 16 +++++++-- src/borg/repository.py | 46 +++++++++++++++++++++--- src/borg/testsuite/repository_test.py | 50 +++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 8 deletions(-) diff --git a/src/borg/cache.py b/src/borg/cache.py index 0f539adbb3..42414cf7f3 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -30,7 +30,7 @@ from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list from .helpers import format_file_size, safe_encode from .helpers import safe_ns -from .helpers import ProgressIndicatorMessage +from .helpers import ProgressIndicatorMessage, ProgressIndicatorPercent from .helpers import msgpack from .helpers.msgpack import int_to_timestamp, timestamp_to_int from .item import ChunkListEntry @@ -810,7 +810,7 @@ def repack_chunkindex(repository): def build_chunkindex_from_repo( - repository, *, slow_rebuild=False, write_immediately=False, init_flags=ChunkIndex.F_USED + repository, *, slow_rebuild=False, write_immediately=False, init_flags=ChunkIndex.F_USED, only_packs=None ): # first, try to build a fresh, mostly complete chunk index from centrally stored index fragments: if not slow_rebuild: @@ -873,15 +873,25 @@ def build_chunkindex_from_repo( # headers and skipping the (much larger) encrypted payloads. Don't call Repository.list() here: # it iterates this same index we are building, so it would recurse. The headers also give each # object's real (chunk_id, offset, size), so every object in a pack is indexed individually. - for info in repository.store_list("packs"): + # only_packs, if given, limits the rebuild to those pack ids; None indexes every pack. + pack_infos = repository.store_list("packs") + if only_packs is not None: + wanted = set(only_packs) + pack_infos = [info for info in pack_infos if hex_to_bin(info.name) in wanted] + pi = ProgressIndicatorPercent( + total=len(pack_infos), msg="Rebuilding chunk index %3.0f%%", msgid="cache.build_chunkindex_from_repo" + ) + for info in pack_infos: # PackReader uses the store directly, so refresh the lock here; a full rebuild can be slow. repository._lock_refresh() + pi.show(increase=1) pack_id = hex_to_bin(info.name) for chunk_id, obj_offset, obj_size in PackReader(repository.store, pack_id).iter_headers(): num_chunks += 1 chunks[chunk_id] = ChunkIndexEntry( flags=init_flags, size=0, pack_id=pack_id, obj_offset=obj_offset, obj_size=obj_size ) + pi.finish() duration = perf_counter() - t0 or 0.001 # Chunk IDs in a list are encoded in 34 bytes: 1 byte msgpack header, 1 byte length, 32 ID bytes. # Protocol overhead is neglected in this calculation. diff --git a/src/borg/repository.py b/src/borg/repository.py index 3bb3e3ed4a..ef6bfe8bad 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -955,9 +955,12 @@ def check(self, repair=False, max_duration=0): The index is hashed first and the packs only if it is intact. The packs could be hashed even with a corrupt index, but a corrupt index already means the user has to repair it, and that rebuild re-reads every pack anyway - so a read-only check just stops and reports it instead of - continuing. The index is never rebuilt here in any case: reading every pack to do so would be - far too slow and expensive for a routine (e.g. cron) check. Salvaging good objects out of - corrupt packs and dropping those packs is left to repair, refs #8572. + continuing. A read-only check never rebuilds the index: reading every pack to do so would be + far too slow and expensive for a routine (e.g. cron) check. + + With repair=True and a corrupt index, the index is rebuilt from the object headers of the packs + whose sha256 still matches, then persisted. Salvaging objects out of corrupt packs is not + implemented yet, refs #8572. """ def verify(namespace, name): @@ -994,6 +997,7 @@ def store_list(namespace): t_last_checkpoint = t_start index_files = index_errors = 0 pack_files = pack_errors = 0 + index_repaired = False # index and packs get separate progress indicators, each running from 0% to 100%. # the index is checked first and in full, on partial checks too: it is small, and index errors # stop the pack check below. @@ -1047,17 +1051,49 @@ def store_list(namespace): logger.info("Finished checking packs.") tracker.clear() pack_pi.finish() + elif repair: + # rebuild the corrupt index from the packs' object headers. only packs whose sha256 still + # matches are used: an entry taken from a corrupted header could point the index at a wrong + # or absent object and break dedup on the next create (refs #8476). chunks that exist only in + # the skipped packs drop out of the index (refs #8572, #10026, salvaging them needs the key). + logger.warning("Repository index is corrupted; rebuilding it from the packs.") + pack_infos = store_list("packs") + good_pack_ids = [] + pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs") + for info in pack_infos: + self._lock_refresh() + pack_pi.show(increase=1) + pack_files += 1 + if verify("packs", info.name): + good_pack_ids.append(hex_to_bin(info.name)) + else: + pack_errors += 1 + if pack_infos: + pack_pi.show(current=len(pack_infos)) # finish at 100% + pack_pi.finish() + from .cache import build_chunkindex_from_repo + + # write_immediately stores the rebuilt index and deletes the corrupt fragments. + build_chunkindex_from_repo(self, slow_rebuild=True, write_immediately=True, only_packs=good_pack_ids) + index_repaired = True else: - # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572). logger.error("Repository index is corrupted and must be repaired; skipping the pack check.") objs_errors = index_errors + pack_errors logger.info( f"Checked {index_files} index files ({index_errors} errors) and {pack_files} packs ({pack_errors} errors)." ) + if index_repaired: + logger.info("Repository index was corrupted and has been rebuilt from the intact packs.") if objs_errors == 0: logger.info(f"Finished {mode} repository check, no problems found.") elif repair: - logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).") + if pack_errors: + logger.error( + f"Finished {mode} repository check, {pack_errors} corrupt pack(s) could not be " + "repaired (pack repair is not implemented yet)." + ) + else: + logger.info(f"Finished {mode} repository check, repaired.") else: logger.error(f"Finished {mode} repository check, errors found.") return objs_errors == 0 or repair diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index 8e20ee8b90..65b713b8ef 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -1053,6 +1053,56 @@ def test_check_detects_index_corruption(tmp_path): assert repository.check(repair=False) is False # mismatch between content hash and name detected +def test_check_repair_rebuilds_corrupt_index(tmp_path): + # check(repair=True) rebuilds a corrupt index from the packs' object headers. + location = os.fspath(tmp_path / "repo") + ids = [H(x) for x in range(10)] + with Repository(location, exclusive=True, create=True) as repository: + for i, cid in enumerate(ids): + repository.put(cid, fchunk(bytes([i]) * 20, chunk_id=cid)) + repository.flush() # seal the pack(s) and let close() persist the index + with reopen(repository) as repository: + index_names = [f"index/{info.name}" for info in repository.store_list("index")] + assert index_names # close() persisted at least one index fragment + for name in index_names: # rot every fragment so its content no longer matches its sha256 name + data = bytearray(repository.store_load(name)) + data[0] ^= 0xFF + repository.store_store(name, bytes(data)) + assert repository.check(repair=False) is False # read-only check reports the corrupt index + with reopen(repository) as repository: + assert repository.check(repair=True) is True # repair rebuilds the index from the packs + with reopen(repository) as repository: + assert repository.check(repair=False) is True # the rebuilt index passes a read-only check + for i, cid in enumerate(ids): + assert pdchunk(repository.get(cid)) == bytes([i]) * 20 # every chunk is indexed and resolves + + +def test_check_repair_excludes_corrupt_pack_from_rebuilt_index(tmp_path): + # The rebuild must not trust a corrupt pack's headers (they could poison the index and break dedup, + # refs #8476), so a pack that fails its sha256 is left out of the rebuilt index; its chunk is dropped. + location = os.fspath(tmp_path / "repo") + with Repository(location, exclusive=True, create=True) as repository: + repository.put(H(1), fchunk(b"GOOD-CHUNK", chunk_id=H(1))) + repository.flush() # seal a pack holding H(1) + repository.put(H(2), fchunk(b"LOST-CHUNK", chunk_id=H(2))) + repository.flush() # seal a separate pack holding H(2) + with reopen(repository) as repository: + bad_pack_name = "packs/" + bin_to_hex(repository.chunks[H(2)].pack_id) + data = bytearray(repository.store_load(bad_pack_name)) + data[-1] ^= 0xFF # rot the pack holding H(2): its content no longer matches its sha256 name + repository.store_store(bad_pack_name, bytes(data)) + for info in repository.store_list("index"): # rot the index so repair takes the rebuild path + name = f"index/{info.name}" + idata = bytearray(repository.store_load(name)) + idata[0] ^= 0xFF + repository.store_store(name, bytes(idata)) + with reopen(repository) as repository: + assert repository.check(repair=True) is True + with reopen(repository) as repository: + assert H(2) not in repository.chunks # corrupt pack's chunk left out of the rebuilt index + assert pdchunk(repository.get(H(1))) == b"GOOD-CHUNK" # intact pack's chunk recovered and resolves + + def test_check_warns_on_invalid_chunk_index(tmp_path, caplog): # check warns about an invalid chunk index but does not fail, since the index is not part of # the repository's object integrity.