From 9266cdcbcc96e3a69e749f59896494a500d858bd Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Wed, 2 Sep 2026 21:12:20 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20Fix=20RecursionError=20on=20?= =?UTF-8?q?long=20runs=20of=20consecutive=20newlines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Splitter._next_mark` skipped newline marks by calling itself recursively, burning one stack frame per consecutive newline. Files with ~1000 or more consecutive lines containing none of `{`, `}`, `"`, `,`, `=` therefore raised a `RecursionError` instead of parsing (or, inside a braced value, aborted parsing with a "please report this bug" message). Newlines are now skipped in an iterative loop, keeping the line counting, char-index and EOF bookkeeping unchanged. Co-Authored-By: Claude Opus 5 --- bibtexparser/splitter.py | 32 +++---- .../test_splitter_many_newlines.py | 86 +++++++++++++++++++ 2 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 tests/splitter_tests/test_splitter_many_newlines.py diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index bbc3e97..1f0b28e 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -102,22 +102,24 @@ def _next_mark(self, accept_eof: bool) -> re.Match | None: self._current_char_index = m.start() return m - # Get next mark from iterator - m = next(self._markiter, None) - if m is not None: + # Iterative: long newline runs would exhaust the recursion limit. + while True: + m = next(self._markiter, None) + if m is None: + # Reached end of file + self._current_char_index = len(self.bibstr) + if not accept_eof: + raise BlockAbortedException( + abort_reason="Unexpectedly reached end of file.", + end_index=self._current_char_index, + ) + return None + self._current_char_index = m.start() - if m.group(0) == "\n": - self._current_line += 1 - return self._next_mark(accept_eof=accept_eof) - else: - # Reached end of file - self._current_char_index = len(self.bibstr) - if not accept_eof: - raise BlockAbortedException( - abort_reason="Unexpectedly reached end of file.", - end_index=self._current_char_index, - ) - return m + if m.group(0) != "\n": + return m + + self._current_line += 1 def _move_to_closed_bracket(self) -> int: """Index of the curly bracket closing a just opened one.""" diff --git a/tests/splitter_tests/test_splitter_many_newlines.py b/tests/splitter_tests/test_splitter_many_newlines.py new file mode 100644 index 0000000..f066223 --- /dev/null +++ b/tests/splitter_tests/test_splitter_many_newlines.py @@ -0,0 +1,86 @@ +"""Tests for bibtex containing long runs of consecutive newlines. + +Skipping newlines used to be implemented recursively, which raised a +`RecursionError` on files containing roughly 1000 or more consecutive lines +without any of the characters `{`, `}`, `"`, `,` or `=`.""" + +import bibtexparser +from bibtexparser.splitter import Splitter + +MANY = 3000 + + +def test_many_blank_lines_between_blocks(): + """Long runs of blank lines between blocks must not abort parsing.""" + bibtex_str = ( + "@article{article1, title={title1}}" + "\n" * MANY + "@article{article2, title={title2}}" + ) + + library = Splitter(bibtex_str).split() + + assert len(library.failed_blocks) == 0 + assert [entry.key for entry in library.entries] == ["article1", "article2"] + + +def test_many_newlines_in_braced_field_value(): + """Long runs of newlines within a braced field value must not abort parsing.""" + bibtex_str = "@article{article1, title={before" + "\n" * MANY + "after}}" + + library = Splitter(bibtex_str).split() + + assert len(library.failed_blocks) == 0 + assert len(library.entries) == 1 + assert library.entries[0].fields_dict["title"].value == "{before" + "\n" * MANY + "after}" + + +def test_many_newlines_in_quoted_field_value(): + """Long runs of newlines within a quoted field value must not abort parsing.""" + bibtex_str = '@article{article1, title="before' + "\n" * MANY + 'after"}' + + library = Splitter(bibtex_str).split() + + assert len(library.failed_blocks) == 0 + assert len(library.entries) == 1 + assert library.entries[0].fields_dict["title"].value == '"before' + "\n" * MANY + 'after"' + + +def test_line_numbers_after_many_blank_lines(): + """Line numbers must remain correct after a large run of blank lines.""" + bibtex_str = "\n" * MANY + "@article{article1,\n title={title1},\n year={2020}\n}" + + library = Splitter(bibtex_str).split() + + assert len(library.failed_blocks) == 0 + entry = library.entries[0] + assert entry.start_line == MANY + assert entry.fields_dict["title"].start_line == MANY + 1 + assert entry.fields_dict["year"].start_line == MANY + 2 + + +def test_many_blank_lines_in_implicit_comment(): + """Long runs of blank lines around an implicit comment must not abort parsing.""" + bibtex_str = ( + "An implicit comment." + + "\n" * MANY + + "Another implicit comment.\n" + + "@article{article1, title={title1}}" + ) + + library = Splitter(bibtex_str).split() + + assert len(library.failed_blocks) == 0 + assert len(library.entries) == 1 + assert library.entries[0].start_line == MANY + 1 + assert [comment.comment for comment in library.comments] == [ + "An implicit comment." + "\n" * MANY + "Another implicit comment." + ] + + +def test_many_blank_lines_at_end_of_file(): + """A file ending in many blank lines must not abort parsing.""" + bibtex_str = "@article{article1, title={title1}}" + "\n" * MANY + + library = bibtexparser.parse_string(bibtex_str) + + assert len(library.failed_blocks) == 0 + assert [entry.key for entry in library.entries] == ["article1"] From 699414c7b4138fac494523741d183b608159cb32 Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Wed, 2 Sep 2026 22:08:20 +0200 Subject: [PATCH 2/2] style: Remove explanatory comment in _next_mark Co-Authored-By: Claude Opus 5 --- bibtexparser/splitter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bibtexparser/splitter.py b/bibtexparser/splitter.py index 1f0b28e..e90b311 100644 --- a/bibtexparser/splitter.py +++ b/bibtexparser/splitter.py @@ -102,7 +102,6 @@ def _next_mark(self, accept_eof: bool) -> re.Match | None: self._current_char_index = m.start() return m - # Iterative: long newline runs would exhaust the recursion limit. while True: m = next(self._markiter, None) if m is None: