Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions src/surreal_memory/extraction/keywords.py
Original file line number Diff line number Diff line change
Expand Up @@ -554,12 +554,13 @@
)

# Clause boundary: bigrams never pair words that cross one of these.
# Em-dash is included — it sets off a parenthetical/separate clause the same as a
# comma (live-proven: "keywords.py — bigramy" otherwise lets the "py" filename
# fragment glue onto the next clause's first word). Plain ASCII hyphen is
# Em-dash (—) and en-dash are both included — each sets off a parenthetical /
# separate clause the same as a comma (live-proven: "keywords.py — bigramy"
# otherwise lets the "py" filename fragment glue onto the next clause's first
# word; editors and OSes emit the en-dash just as often). Plain ASCII hyphen is
# deliberately NOT included: it's mid-word in compounds like "write-gate", which
# must still yield the useful bigram "write gate".
_CLAUSE_BOUNDARY = re.compile(r"[.,;:!?\n\r—]+")
_CLAUSE_BOUNDARY = re.compile(r"[.,;:!?\n\r—]+") # noqa: RUF001 (literal en-dash intended)


def _get_stop_words(language: str, text: str) -> frozenset[str]:
Expand Down
19 changes: 19 additions & 0 deletions tests/unit/test_polish_keywords.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,25 @@ def test_no_bigram_across_em_dash(self) -> None:
assert "py" in unigrams
assert "bigramy" in unigrams

def test_no_bigram_across_en_dash(self) -> None:
# The en-dash (U+2013) is a clause boundary too, not only the em-dash
# (—, U+2014): editors and OSes emit it for ranges and asides, so without
# it the same cross-clause junk bigram (a filename fragment glued onto the
# next clause) slips through.
text = "blad w keywords.py – bigramy nie powinny przecinac granic" # noqa: RUF001
bigrams = _bigrams(text)
assert "py bigramy" not in bigrams
unigrams = _unigrams(text)
assert "py" in unigrams
assert "bigramy" in unigrams

def test_hyphen_is_not_a_clause_boundary(self) -> None:
# A hyphen inside a compound term must NOT split a clause: the two halves
# are adjacent content words and should still pair as a bigram (contrast
# with the en-/em-dash, which do split).
bigrams = _bigrams("modul cache-aside dziala szybko")
assert "cache aside" in bigrams

def test_phase1_junk_bigrams_eliminated(self) -> None:
junk = {
"reguly czy",
Expand Down
Loading