From 94d840a97f61fa8922735dda79973b3b48ab69d6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 28 Sep 2026 20:26:07 +0200 Subject: [PATCH] remote: use deques for the RPC call queues, fixes #10450 RemoteRepository.call_many took the next call from the front of a list (calls.pop(0), self.preload_ids.pop(0)), which moves all the remaining entries each time. For a single item with millions of chunks (preload + get_many over all of its chunk ids), that made extract from a remote repository quadratic in the chunk count of the item. Use collections.deque with popleft() for calls, waiting_for and preload_ids. Co-Authored-By: Claude Opus 5.5 --- src/borg/remote.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index d29f1d9936..458e475526 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -13,6 +13,7 @@ import textwrap import time import traceback +from collections import deque from subprocess import Popen, PIPE, TimeoutExpired from . import __version__ @@ -557,7 +558,7 @@ def required_version(self): def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock=True, append_only=False, make_parent_dirs=False, args=None): self.location = self._location = location - self.preload_ids = [] + self.preload_ids = deque() self.msgid = 0 self.rx_bytes = 0 self.tx_bytes = 0 @@ -836,8 +837,8 @@ def handle_error(unpacked): else: raise self.RPCError(unpacked) - calls = list(calls) - waiting_for = [] + calls = deque(calls) + waiting_for = deque() maximum_to_send = 0 if wait else self.upload_buffer_size_limit send_buffer() # Try to send data, as some cases (async_response) will never try to send data otherwise. try: @@ -851,7 +852,7 @@ def handle_error(unpacked): while waiting_for: try: unpacked = self.responses.pop(waiting_for[0]) - waiting_for.pop(0) + waiting_for.popleft() if b'exception_class' in unpacked: handle_error(unpacked) else: @@ -926,9 +927,9 @@ def handle_error(unpacked): if is_preloaded: assert cmd == 'get', "is_preload is only supported for 'get'" if calls[0]['id'] in self.chunkid_to_msgids: - waiting_for.append(pop_preload_msgid(calls.pop(0)['id'])) + waiting_for.append(pop_preload_msgid(calls.popleft()['id'])) else: - args = calls.pop(0) + args = calls.popleft() if cmd == 'get' and args['id'] in self.chunkid_to_msgids: waiting_for.append(pop_preload_msgid(args['id'])) else: @@ -939,7 +940,7 @@ def handle_error(unpacked): else: self.to_send.push_back(msgpack.packb((1, self.msgid, cmd, self.named_to_positional(cmd, args)))) if not self.to_send and self.preload_ids: - chunk_id = self.preload_ids.pop(0) + chunk_id = self.preload_ids.popleft() args = {'id': chunk_id} self.msgid += 1 self.chunkid_to_msgids.setdefault(chunk_id, []).append(self.msgid) @@ -1088,7 +1089,7 @@ def async_response(self, wait=True): return resp def preload(self, ids): - self.preload_ids += ids + self.preload_ids.extend(ids) def handle_remote_line(line):