From 5648082d0b0eebdb4f55dcd078c73375319d9379 Mon Sep 17 00:00:00 2001 From: Eddy Zhang Date: Sun, 23 Aug 2026 11:23:51 +1000 Subject: [PATCH 1/3] fix: merge a short trailing split in EmbeddingBasedDocumentSplitter `_merge_small_splits` only merges forward: it folds the next split into a running accumulator while that accumulator is below `min_length`. Whatever is left in the accumulator when the loop ends is appended unconditionally, so a final split shorter than `min_length` is emitted as its own document, breaking the documented `min_length` promise. Merge that trailing split into its predecessor instead, subject to the same `max_length` limit that already governs forward merges, so a blocked merge keeps behaving as it does today. Fixes #12436 --- .../embedding_based_document_splitter.py | 11 +++++++ ...short-trailing-split-8f5b640fdbc3ccbe.yaml | 7 +++++ .../test_embedding_based_document_splitter.py | 30 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml diff --git a/haystack/components/preprocessors/embedding_based_document_splitter.py b/haystack/components/preprocessors/embedding_based_document_splitter.py index 17acefd3b7..092136a707 100644 --- a/haystack/components/preprocessors/embedding_based_document_splitter.py +++ b/haystack/components/preprocessors/embedding_based_document_splitter.py @@ -445,6 +445,17 @@ def _merge_small_splits(self, splits: list[str]) -> list[str]: # Don't forget the last split merged.append(current_split) + # The loop above only merges forward, so the final split never had a chance to absorb anything and can + # still be below min_length. Merge it into its predecessor instead, subject to the same max_length limit + # that governs forward merges. + if ( + len(merged) > 1 + and len(merged[-1]) < self.min_length + and len(merged[-2]) + len(merged[-1]) < self.max_length + ): + trailing_split = merged.pop() + merged[-1] += trailing_split + return merged def _split_large_splits(self, splits: list[str]) -> list[str]: diff --git a/releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml b/releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml new file mode 100644 index 0000000000..82116c99ac --- /dev/null +++ b/releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixed `EmbeddingBasedDocumentSplitter` emitting a final split shorter than `min_length`. Small splits were + only merged forward, so the last one had nothing left to absorb and was returned as its own document. It is + now merged into the preceding split, unless doing so would reach `max_length` — the same limit that already + governs forward merges. diff --git a/test/components/preprocessors/test_embedding_based_document_splitter.py b/test/components/preprocessors/test_embedding_based_document_splitter.py index 50db2a8aa8..968a770e35 100644 --- a/test/components/preprocessors/test_embedding_based_document_splitter.py +++ b/test/components/preprocessors/test_embedding_based_document_splitter.py @@ -215,6 +215,36 @@ def test_merge_small_splits_respect_max_length(self): # Second split is merged with third split to get above min_length and still beneath max_length assert merged[1] == "1234567891234" + def test_merge_small_splits_merges_short_trailing_split(self): + mock_embedder = Mock() + splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, min_length=10) + + # The loop only merges forward, so the final accumulator has nothing left to absorb. + splits = ["Long enough text ", "Ok."] + merged = splitter._merge_small_splits(splits=splits) + + assert merged == ["Long enough text Ok."] + + def test_merge_small_splits_keeps_short_trailing_split_when_max_length_blocks(self): + mock_embedder = Mock() + splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, min_length=10, max_length=15) + + # Merging backwards would reach max_length, so the short tail stays on its own, + # matching how a blocked forward merge already behaves. + splits = ["123456789012", "1234"] + merged = splitter._merge_small_splits(splits=splits) + + assert merged == ["123456789012", "1234"] + + def test_merge_small_splits_keeps_a_lone_short_split(self): + mock_embedder = Mock() + splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, min_length=10) + + # Nothing to merge into. + merged = splitter._merge_small_splits(splits=["Ok."]) + + assert merged == ["Ok."] + def test_create_documents_from_splits(self): mock_embedder = Mock() splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder) From b7fe5aa25930362236deb00497292a586f4c2e31 Mon Sep 17 00:00:00 2001 From: Eddy Zhang Date: Sun, 23 Aug 2026 11:45:41 +1000 Subject: [PATCH 2/3] docs: use reStructuredText inline code in the release note AGENTS.md specifies reStructuredText for release notes, so inline code takes double backticks. Also drops the em dash so the file stays ASCII, matching every other note in releasenotes/notes. --- .../merge-short-trailing-split-8f5b640fdbc3ccbe.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml b/releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml index 82116c99ac..456a4d4603 100644 --- a/releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml +++ b/releasenotes/notes/merge-short-trailing-split-8f5b640fdbc3ccbe.yaml @@ -1,7 +1,7 @@ --- fixes: - | - Fixed `EmbeddingBasedDocumentSplitter` emitting a final split shorter than `min_length`. Small splits were - only merged forward, so the last one had nothing left to absorb and was returned as its own document. It is - now merged into the preceding split, unless doing so would reach `max_length` — the same limit that already - governs forward merges. + Fixed ``EmbeddingBasedDocumentSplitter`` emitting a final split shorter than ``min_length``. Small splits + were only merged forward, so the last one had nothing left to absorb and was returned as its own document. + It is now merged into the preceding split, unless doing so would reach ``max_length``, the same limit that + already governs forward merges. From fdfcc57ea822f26343707f2689efe5104836c482 Mon Sep 17 00:00:00 2001 From: Eddy Zhang Date: Mon, 24 Aug 2026 17:31:11 +1000 Subject: [PATCH 3/3] docs: shorten the backward-merge comment per review --- .../preprocessors/embedding_based_document_splitter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/haystack/components/preprocessors/embedding_based_document_splitter.py b/haystack/components/preprocessors/embedding_based_document_splitter.py index 092136a707..34e74cfa09 100644 --- a/haystack/components/preprocessors/embedding_based_document_splitter.py +++ b/haystack/components/preprocessors/embedding_based_document_splitter.py @@ -445,9 +445,8 @@ def _merge_small_splits(self, splits: list[str]) -> list[str]: # Don't forget the last split merged.append(current_split) - # The loop above only merges forward, so the final split never had a chance to absorb anything and can - # still be below min_length. Merge it into its predecessor instead, subject to the same max_length limit - # that governs forward merges. + # The loop only merges forward, so the final split can still be below min_length. Merge it backwards, + # subject to the same max_length limit as forward merges. if ( len(merged) > 1 and len(merged[-1]) < self.min_length