check --repair: rebuild a corrupt repository index from the packs, #10026

src/borg/cache.py

@@ -30,7 +30,7 @@
 from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list
 from .helpers import format_file_size, safe_encode
 from .helpers import safe_ns
-from .helpers import ProgressIndicatorMessage
+from .helpers import ProgressIndicatorMessage, ProgressIndicatorPercent
 from .helpers import msgpack
 from .helpers.msgpack import int_to_timestamp, timestamp_to_int
 from .item import ChunkListEntry
@@ -810,7 +810,7 @@ def repack_chunkindex(repository):
 
 
 def build_chunkindex_from_repo(
-    repository, *, slow_rebuild=False, write_immediately=False, init_flags=ChunkIndex.F_USED
+    repository, *, slow_rebuild=False, write_immediately=False, init_flags=ChunkIndex.F_USED, only_packs=None
 ):
     # first, try to build a fresh, mostly complete chunk index from centrally stored index fragments:
     if not slow_rebuild:
@@ -873,15 +873,25 @@ def build_chunkindex_from_repo(
     # headers and skipping the (much larger) encrypted payloads. Don't call Repository.list() here:
     # it iterates this same index we are building, so it would recurse. The headers also give each
     # object's real (chunk_id, offset, size), so every object in a pack is indexed individually.
-    for info in repository.store_list("packs"):
+    # only_packs, if given, limits the rebuild to those pack ids; None indexes every pack.
+    pack_infos = repository.store_list("packs")
+    if only_packs is not None:
+        wanted = set(only_packs)
+        pack_infos = [info for info in pack_infos if hex_to_bin(info.name) in wanted]
+    pi = ProgressIndicatorPercent(
+        total=len(pack_infos), msg="Rebuilding chunk index %3.0f%%", msgid="cache.build_chunkindex_from_repo"
+    )
+    for info in pack_infos:
         # PackReader uses the store directly, so refresh the lock here; a full rebuild can be slow.
         repository._lock_refresh()
+        pi.show(increase=1)
         pack_id = hex_to_bin(info.name)
         for chunk_id, obj_offset, obj_size in PackReader(repository.store, pack_id).iter_headers():
             num_chunks += 1
             chunks[chunk_id] = ChunkIndexEntry(
                 flags=init_flags, size=0, pack_id=pack_id, obj_offset=obj_offset, obj_size=obj_size
             )
+    pi.finish()
     duration = perf_counter() - t0 or 0.001
     # Chunk IDs in a list are encoded in 34 bytes: 1 byte msgpack header, 1 byte length, 32 ID bytes.
     # Protocol overhead is neglected in this calculation.

src/borg/repository.py

@@ -955,9 +955,12 @@ def check(self, repair=False, max_duration=0):
         The index is hashed first and the packs only if it is intact. The packs could be hashed even
         with a corrupt index, but a corrupt index already means the user has to repair it, and that
         rebuild re-reads every pack anyway - so a read-only check just stops and reports it instead of
-        continuing. The index is never rebuilt here in any case: reading every pack to do so would be
-        far too slow and expensive for a routine (e.g. cron) check. Salvaging good objects out of
-        corrupt packs and dropping those packs is left to repair, refs #8572.
+        continuing. A read-only check never rebuilds the index: reading every pack to do so would be
+        far too slow and expensive for a routine (e.g. cron) check.
+
+        With repair=True and a corrupt index, the index is rebuilt from the object headers of the packs
+        whose sha256 still matches, then persisted. Salvaging objects out of corrupt packs is not
+        implemented yet, refs #8572.
         """
 
         def verify(namespace, name):
@@ -994,6 +997,7 @@ def store_list(namespace):
         t_last_checkpoint = t_start
         index_files = index_errors = 0
         pack_files = pack_errors = 0
+        index_repaired = False
         # index and packs get separate progress indicators, each running from 0% to 100%.
         # the index is checked first and in full, on partial checks too: it is small, and index errors
         # stop the pack check below.
@@ -1047,17 +1051,49 @@ def store_list(namespace):
                 logger.info("Finished checking packs.")
                 tracker.clear()
             pack_pi.finish()
+        elif repair:
+            # rebuild the corrupt index from the packs' object headers. only packs whose sha256 still
+            # matches are used: an entry taken from a corrupted header could point the index at a wrong
+            # or absent object and break dedup on the next create (refs #8476). chunks that exist only in
+            # the skipped packs drop out of the index (refs #8572, #10026, salvaging them needs the key).
+            logger.warning("Repository index is corrupted; rebuilding it from the packs.")
+            pack_infos = store_list("packs")
+            good_pack_ids = []
+            pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs")
+            for info in pack_infos:
+                self._lock_refresh()
+                pack_pi.show(increase=1)
+                pack_files += 1
+                if verify("packs", info.name):
+                    good_pack_ids.append(hex_to_bin(info.name))
+                else:
+                    pack_errors += 1
+            if pack_infos:
+                pack_pi.show(current=len(pack_infos))  # finish at 100%
+            pack_pi.finish()
+            from .cache import build_chunkindex_from_repo
+
+            # write_immediately stores the rebuilt index and deletes the corrupt fragments.
+            build_chunkindex_from_repo(self, slow_rebuild=True, write_immediately=True, only_packs=good_pack_ids)
+            index_repaired = True
         else:
-            # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
         objs_errors = index_errors + pack_errors
         logger.info(
             f"Checked {index_files} index files ({index_errors} errors) and {pack_files} packs ({pack_errors} errors)."
         )
+        if index_repaired:
+            logger.info("Repository index was corrupted and has been rebuilt from the intact packs.")
         if objs_errors == 0:
             logger.info(f"Finished {mode} repository check, no problems found.")
         elif repair:
-            logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).")
+            if pack_errors:
+                logger.error(
+                    f"Finished {mode} repository check, {pack_errors} corrupt pack(s) could not be "
+                    "repaired (pack repair is not implemented yet)."
+                )
+            else:
+                logger.info(f"Finished {mode} repository check, repaired.")
         else:
             logger.error(f"Finished {mode} repository check, errors found.")
         return objs_errors == 0 or repair

src/borg/testsuite/repository_test.py

@@ -1053,6 +1053,56 @@ def test_check_detects_index_corruption(tmp_path):
         assert repository.check(repair=False) is False  # mismatch between content hash and name detected
 
 
+def test_check_repair_rebuilds_corrupt_index(tmp_path):
+    # check(repair=True) rebuilds a corrupt index from the packs' object headers.
+    location = os.fspath(tmp_path / "repo")
+    ids = [H(x) for x in range(10)]
+    with Repository(location, exclusive=True, create=True) as repository:
+        for i, cid in enumerate(ids):
+            repository.put(cid, fchunk(bytes([i]) * 20, chunk_id=cid))
+        repository.flush()  # seal the pack(s) and let close() persist the index
+    with reopen(repository) as repository:
+        index_names = [f"index/{info.name}" for info in repository.store_list("index")]
+        assert index_names  # close() persisted at least one index fragment
+        for name in index_names:  # rot every fragment so its content no longer matches its sha256 name
+            data = bytearray(repository.store_load(name))
+            data[0] ^= 0xFF
+            repository.store_store(name, bytes(data))
+        assert repository.check(repair=False) is False  # read-only check reports the corrupt index
+    with reopen(repository) as repository:
+        assert repository.check(repair=True) is True  # repair rebuilds the index from the packs
+    with reopen(repository) as repository:
+        assert repository.check(repair=False) is True  # the rebuilt index passes a read-only check
+        for i, cid in enumerate(ids):
+            assert pdchunk(repository.get(cid)) == bytes([i]) * 20  # every chunk is indexed and resolves
+
+
+def test_check_repair_excludes_corrupt_pack_from_rebuilt_index(tmp_path):
+    # The rebuild must not trust a corrupt pack's headers (they could poison the index and break dedup,
+    # refs #8476), so a pack that fails its sha256 is left out of the rebuilt index; its chunk is dropped.
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        repository.put(H(1), fchunk(b"GOOD-CHUNK", chunk_id=H(1)))
+        repository.flush()  # seal a pack holding H(1)
+        repository.put(H(2), fchunk(b"LOST-CHUNK", chunk_id=H(2)))
+        repository.flush()  # seal a separate pack holding H(2)
+    with reopen(repository) as repository:
+        bad_pack_name = "packs/" + bin_to_hex(repository.chunks[H(2)].pack_id)
+        data = bytearray(repository.store_load(bad_pack_name))
+        data[-1] ^= 0xFF  # rot the pack holding H(2): its content no longer matches its sha256 name
+        repository.store_store(bad_pack_name, bytes(data))
+        for info in repository.store_list("index"):  # rot the index so repair takes the rebuild path
+            name = f"index/{info.name}"
+            idata = bytearray(repository.store_load(name))
+            idata[0] ^= 0xFF
+            repository.store_store(name, bytes(idata))
+    with reopen(repository) as repository:
+        assert repository.check(repair=True) is True
+    with reopen(repository) as repository:
+        assert H(2) not in repository.chunks  # corrupt pack's chunk left out of the rebuilt index
+        assert pdchunk(repository.get(H(1))) == b"GOOD-CHUNK"  # intact pack's chunk recovered and resolves
+
+
 def test_check_warns_on_invalid_chunk_index(tmp_path, caplog):
     # check warns about an invalid chunk index but does not fail, since the index is not part of
     # the repository's object integrity.