diff --git a/services/kb_service.py b/services/kb_service.py index 05ca384..1150da5 100644 --- a/services/kb_service.py +++ b/services/kb_service.py @@ -56,7 +56,8 @@ def _chunk_text(text: str) -> list[str]: current = f"{current}\n{piece}" else: chunks.append(current) - tail = current[-CHUNK_OVERLAP:] if CHUNK_OVERLAP else "" + overlap = min(CHUNK_OVERLAP, max(0, CHUNK_SIZE - len(piece) - 1)) + tail = current[-overlap:] if overlap else "" current = (tail + "\n" + piece).strip() if tail else piece if current: chunks.append(current) @@ -69,6 +70,8 @@ def _split_long(text: str) -> list[str]: current = "" for s in sentences: if len(s) > CHUNK_SIZE: + if current: + out.append(current) # Hard-split an oversized sentence on word boundaries. for i in range(0, len(s), CHUNK_SIZE): out.append(s[i : i + CHUNK_SIZE]) diff --git a/tests/unit/services/test_kb_service.py b/tests/unit/services/test_kb_service.py index f280ccb..7f996f9 100644 --- a/tests/unit/services/test_kb_service.py +++ b/tests/unit/services/test_kb_service.py @@ -38,6 +38,21 @@ def test_split_long_hard_splits_oversized_sentence(): assert "".join(pieces) == "X" * 1000 +def test_split_long_preserves_buffer_before_oversized_sentence(): + prefix = "Warranty lasts two years." + oversized = "X" * (kb.CHUNK_SIZE + 25) + text = f"{prefix} {oversized}" + + pieces = kb._split_long(text) + chunks = kb._chunk_text(text) + + assert pieces[0] == prefix + assert "".join(pieces[1:]) == oversized + assert all(len(piece) <= kb.CHUNK_SIZE for piece in pieces) + assert any(prefix in chunk for chunk in chunks) + assert all(len(chunk) <= kb.CHUNK_SIZE for chunk in chunks) + + # --- build_context_block ----------------------------------------------------- def test_context_block_empty(): assert kb.build_context_block([]) == ""