diff --git a/bibtexparser/library.py b/bibtexparser/library.py index 6454276..e47869a 100644 --- a/bibtexparser/library.py +++ b/bibtexparser/library.py @@ -62,18 +62,25 @@ def add(self, blocks: list[Block] | Block, fail_on_duplicate_key: bool = True): def _find_duplicate_keys(self, blocks: list[Block]) -> list[str]: """Keys of blocks that would become duplicates when added to the library.""" + # Look keys up in the by-key dicts directly (rather than copying them into + # sets), so that the cost depends only on the number of blocks being added, + # not on the size of the library. duplicate_keys = [] - seen_entry_keys = set(self._entries_by_key) - seen_string_keys = set(self._strings_by_key) + new_entry_keys = set() + new_string_keys = set() for block in blocks: if isinstance(block, Entry): - if block.key in seen_entry_keys: - duplicate_keys.append(block.key) - seen_entry_keys.add(block.key) + key = block.key + if key in self._entries_by_key or key in new_entry_keys: + duplicate_keys.append(key) + else: + new_entry_keys.add(key) elif isinstance(block, String): - if block.key in seen_string_keys: - duplicate_keys.append(block.key) - seen_string_keys.add(block.key) + key = block.key + if key in self._strings_by_key or key in new_string_keys: + duplicate_keys.append(key) + else: + new_string_keys.add(key) return duplicate_keys def _block_index(self, block: Block, skip: set[int] | None = None) -> int: diff --git a/bibtexparser/middlewares/enclosing.py b/bibtexparser/middlewares/enclosing.py index 1393fae..91a85e4 100644 --- a/bibtexparser/middlewares/enclosing.py +++ b/bibtexparser/middlewares/enclosing.py @@ -121,8 +121,11 @@ def _is_enclosed_in_quotes(value: str) -> bool: concatenation expression `"intro" # "outro"`. Escaped quotes and quotes inside braces are not counted. """ - depth = 0 last_index = len(value) - 1 + if value.find('"', 1, last_index) < 0: + # Fast path for the common case of a value without inner quotes. + return True + depth = 0 for match in _BRACES_AND_QUOTE.finditer(value): index = match.start() if index == 0 or index == last_index: diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index 7aff629..a6b5ae9 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -308,16 +308,16 @@ def split(self) -> Library: # Clean up previous block implicit_comment implicit_comment = self._end_implicit_comment(m.start()) if implicit_comment is not None: - library.add(implicit_comment) + library.add(implicit_comment, fail_on_duplicate_key=False) self._implicit_comment_start = None start_line = self._current_line try: # Start new block parsing if m_val.startswith("@comment"): - library.add(self._handle_explicit_comment()) + library.add(self._handle_explicit_comment(), fail_on_duplicate_key=False) elif m_val.startswith("@preamble"): - library.add(self._handle_preamble()) + library.add(self._handle_preamble(), fail_on_duplicate_key=False) elif m_val.startswith("@string"): library.add(self._handle_string(m), fail_on_duplicate_key=False) else: @@ -338,7 +338,8 @@ def split(self) -> Library: start_line=start_line, raw=self.bibstr[m.start() : e.end_index], error=e, - ) + ), + fail_on_duplicate_key=False, ) except ParserStateException as e: @@ -365,7 +366,7 @@ def split(self) -> Library: if self._implicit_comment_start is not None: comment = self._end_implicit_comment(len(self.bibstr)) if comment is not None: - library.add(comment) + library.add(comment, fail_on_duplicate_key=False) return library diff --git a/tests/test_library.py b/tests/test_library.py index 1e40ee9..2592a5d 100644 --- a/tests/test_library.py +++ b/tests/test_library.py @@ -218,3 +218,37 @@ def test_replace_entry_with_changed_key(): assert library.blocks == [replacement] assert library.entries == [replacement] assert library.entries_dict == {"replacementKey": replacement} + + +class _NonIterableDict(dict): + """A dict that supports lookups but refuses to be iterated or copied. + + Used to make sure that `Library.add` does not walk the whole by-key dicts + (i.e., does not scale with the library size) when checking for duplicates.""" + + def __iter__(self): + raise AssertionError("by-key dict must not be iterated on add") + + def keys(self): + raise AssertionError("by-key dict must not be iterated on add") + + +def test_add_duplicate_check_does_not_iterate_library(): + """Checking for duplicates must not iterate over all existing keys (was O(library size)).""" + library = Library() + library.add([get_dummy_entry_with_key("existing"), String(key="s", value="{v}")]) + library._entries_by_key = _NonIterableDict(library._entries_by_key) + library._strings_by_key = _NonIterableDict(library._strings_by_key) + + library.add(ImplicitComment(comment="a comment")) + library.add(get_dummy_entry_with_key("new")) + library.add(String(key="t", value="{w}")) + assert len(library.blocks) == 5 + + with pytest.raises(ValueError): + library.add(get_dummy_entry_with_key("existing")) + with pytest.raises(ValueError): + library.add(String(key="s", value="{v}")) + with pytest.raises(ValueError): + library.add([get_dummy_entry_with_key("twice"), get_dummy_entry_with_key("twice")]) + assert len(library.blocks) == 5