From 84ccb956fb4d5a0c2c459d6721b3bfc28a37e98e Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Thu, 3 Sep 2026 21:25:58 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9A=A1=20Make=20Library.add=20duplicate?= =?UTF-8?q?=20check=20independent=20of=20library=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_find_duplicate_keys` copied all existing entry and string keys into sets on every call, making `add()` O(library size) per call and parsing quadratic in the number of comment/preamble/failed blocks (the splitter added those with the default `fail_on_duplicate_key=True`). Look keys up in the by-key dicts directly and pass `fail_on_duplicate_key=False` for all blocks added by the splitter. Co-Authored-By: Claude Fable 5.1 --- bibtexparser/library.py | 23 +++++++++++++++-------- bibtexparser/splitter.py | 11 ++++++----- tests/test_library.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/bibtexparser/library.py b/bibtexparser/library.py index 6454276..e47869a 100644 --- a/bibtexparser/library.py +++ b/bibtexparser/library.py @@ -62,18 +62,25 @@ def add(self, blocks: list[Block] | Block, fail_on_duplicate_key: bool = True): def _find_duplicate_keys(self, blocks: list[Block]) -> list[str]: """Keys of blocks that would become duplicates when added to the library.""" + # Look keys up in the by-key dicts directly (rather than copying them into + # sets), so that the cost depends only on the number of blocks being added, + # not on the size of the library. duplicate_keys = [] - seen_entry_keys = set(self._entries_by_key) - seen_string_keys = set(self._strings_by_key) + new_entry_keys = set() + new_string_keys = set() for block in blocks: if isinstance(block, Entry): - if block.key in seen_entry_keys: - duplicate_keys.append(block.key) - seen_entry_keys.add(block.key) + key = block.key + if key in self._entries_by_key or key in new_entry_keys: + duplicate_keys.append(key) + else: + new_entry_keys.add(key) elif isinstance(block, String): - if block.key in seen_string_keys: - duplicate_keys.append(block.key) - seen_string_keys.add(block.key) + key = block.key + if key in self._strings_by_key or key in new_string_keys: + duplicate_keys.append(key) + else: + new_string_keys.add(key) return duplicate_keys def _block_index(self, block: Block, skip: set[int] | None = None) -> int: diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index 7aff629..a6b5ae9 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -308,16 +308,16 @@ def split(self) -> Library: # Clean up previous block implicit_comment implicit_comment = self._end_implicit_comment(m.start()) if implicit_comment is not None: - library.add(implicit_comment) + library.add(implicit_comment, fail_on_duplicate_key=False) self._implicit_comment_start = None start_line = self._current_line try: # Start new block parsing if m_val.startswith("@comment"): - library.add(self._handle_explicit_comment()) + library.add(self._handle_explicit_comment(), fail_on_duplicate_key=False) elif m_val.startswith("@preamble"): - library.add(self._handle_preamble()) + library.add(self._handle_preamble(), fail_on_duplicate_key=False) elif m_val.startswith("@string"): library.add(self._handle_string(m), fail_on_duplicate_key=False) else: @@ -338,7 +338,8 @@ def split(self) -> Library: start_line=start_line, raw=self.bibstr[m.start() : e.end_index], error=e, - ) + ), + fail_on_duplicate_key=False, ) except ParserStateException as e: @@ -365,7 +366,7 @@ def split(self) -> Library: if self._implicit_comment_start is not None: comment = self._end_implicit_comment(len(self.bibstr)) if comment is not None: - library.add(comment) + library.add(comment, fail_on_duplicate_key=False) return library diff --git a/tests/test_library.py b/tests/test_library.py index 1e40ee9..2592a5d 100644 --- a/tests/test_library.py +++ b/tests/test_library.py @@ -218,3 +218,37 @@ def test_replace_entry_with_changed_key(): assert library.blocks == [replacement] assert library.entries == [replacement] assert library.entries_dict == {"replacementKey": replacement} + + +class _NonIterableDict(dict): + """A dict that supports lookups but refuses to be iterated or copied. + + Used to make sure that `Library.add` does not walk the whole by-key dicts + (i.e., does not scale with the library size) when checking for duplicates.""" + + def __iter__(self): + raise AssertionError("by-key dict must not be iterated on add") + + def keys(self): + raise AssertionError("by-key dict must not be iterated on add") + + +def test_add_duplicate_check_does_not_iterate_library(): + """Checking for duplicates must not iterate over all existing keys (was O(library size)).""" + library = Library() + library.add([get_dummy_entry_with_key("existing"), String(key="s", value="{v}")]) + library._entries_by_key = _NonIterableDict(library._entries_by_key) + library._strings_by_key = _NonIterableDict(library._strings_by_key) + + library.add(ImplicitComment(comment="a comment")) + library.add(get_dummy_entry_with_key("new")) + library.add(String(key="t", value="{w}")) + assert len(library.blocks) == 5 + + with pytest.raises(ValueError): + library.add(get_dummy_entry_with_key("existing")) + with pytest.raises(ValueError): + library.add(String(key="s", value="{v}")) + with pytest.raises(ValueError): + library.add([get_dummy_entry_with_key("twice"), get_dummy_entry_with_key("twice")]) + assert len(library.blocks) == 5 From 80792c161195370f7287fa9e876d3141774b500c Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Thu, 3 Sep 2026 21:25:58 +0200 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9A=A1=20Add=20fast=20path=20for=20quote?= =?UTF-8?q?-enclosed=20values=20in=20RemoveEnclosingMiddleware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_is_enclosed_in_quotes` (added in #604) scanned every quoted value with a regex, making the default parse stack ~4x slower on files with quoted values. Skip the scan when the value contains no inner quote, and avoid the slice copy in the corresponding brace fast path. Co-Authored-By: Claude Fable 5.1 --- bibtexparser/middlewares/enclosing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bibtexparser/middlewares/enclosing.py b/bibtexparser/middlewares/enclosing.py index 1393fae..91a85e4 100644 --- a/bibtexparser/middlewares/enclosing.py +++ b/bibtexparser/middlewares/enclosing.py @@ -121,8 +121,11 @@ def _is_enclosed_in_quotes(value: str) -> bool: concatenation expression `"intro" # "outro"`. Escaped quotes and quotes inside braces are not counted. """ - depth = 0 last_index = len(value) - 1 + if value.find('"', 1, last_index) < 0: + # Fast path for the common case of a value without inner quotes. + return True + depth = 0 for match in _BRACES_AND_QUOTE.finditer(value): index = match.start() if index == 0 or index == last_index: