From d3aafdcc9476a50aa15b27fcac9f2b7df73fe645 Mon Sep 17 00:00:00 2001 From: Robert Sigmundsson Date: Sun, 12 Jul 2026 09:16:53 +0200 Subject: [PATCH] fix(extraction): treat the en-dash as a clause boundary too, and pin hyphen non-boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clause-boundary set had the em-dash (—) but not the en-dash (–, U+2013), which editors and operating systems emit just as often for ranges/asides. Without it the same cross-clause junk bigram the em-dash fix targets slips through when the separator happens to be an en-dash. Add the en-dash to _CLAUSE_BOUNDARY and add two regression tests: one that no bigram bridges an en-dash, and one that a plain ASCII hyphen is NOT a clause boundary (compound halves like "cache-aside" must still pair). --- src/surreal_memory/extraction/keywords.py | 9 +++++---- tests/unit/test_polish_keywords.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/surreal_memory/extraction/keywords.py b/src/surreal_memory/extraction/keywords.py index 1a3ec330..e84b2153 100755 --- a/src/surreal_memory/extraction/keywords.py +++ b/src/surreal_memory/extraction/keywords.py @@ -554,12 +554,13 @@ ) # Clause boundary: bigrams never pair words that cross one of these. -# Em-dash is included — it sets off a parenthetical/separate clause the same as a -# comma (live-proven: "keywords.py — bigramy" otherwise lets the "py" filename -# fragment glue onto the next clause's first word). Plain ASCII hyphen is +# Em-dash (—) and en-dash are both included — each sets off a parenthetical / +# separate clause the same as a comma (live-proven: "keywords.py — bigramy" +# otherwise lets the "py" filename fragment glue onto the next clause's first +# word; editors and OSes emit the en-dash just as often). Plain ASCII hyphen is # deliberately NOT included: it's mid-word in compounds like "write-gate", which # must still yield the useful bigram "write gate". -_CLAUSE_BOUNDARY = re.compile(r"[.,;:!?\n\r—]+") +_CLAUSE_BOUNDARY = re.compile(r"[.,;:!?\n\r—–]+") # noqa: RUF001 (literal en-dash intended) def _get_stop_words(language: str, text: str) -> frozenset[str]: diff --git a/tests/unit/test_polish_keywords.py b/tests/unit/test_polish_keywords.py index 6b516ebd..fb36ba3f 100644 --- a/tests/unit/test_polish_keywords.py +++ b/tests/unit/test_polish_keywords.py @@ -70,6 +70,25 @@ def test_no_bigram_across_em_dash(self) -> None: assert "py" in unigrams assert "bigramy" in unigrams + def test_no_bigram_across_en_dash(self) -> None: + # The en-dash (U+2013) is a clause boundary too, not only the em-dash + # (—, U+2014): editors and OSes emit it for ranges and asides, so without + # it the same cross-clause junk bigram (a filename fragment glued onto the + # next clause) slips through. + text = "blad w keywords.py – bigramy nie powinny przecinac granic" # noqa: RUF001 + bigrams = _bigrams(text) + assert "py bigramy" not in bigrams + unigrams = _unigrams(text) + assert "py" in unigrams + assert "bigramy" in unigrams + + def test_hyphen_is_not_a_clause_boundary(self) -> None: + # A hyphen inside a compound term must NOT split a clause: the two halves + # are adjacent content words and should still pair as a bigram (contrast + # with the en-/em-dash, which do split). + bigrams = _bigrams("modul cache-aside dziala szybko") + assert "cache aside" in bigrams + def test_phase1_junk_bigrams_eliminated(self) -> None: junk = { "reguly czy",