diff --git a/haystack/components/preprocessors/markdown_header_splitter.py b/haystack/components/preprocessors/markdown_header_splitter.py index 19c20961be..63b91ad21c 100644 --- a/haystack/components/preprocessors/markdown_header_splitter.py +++ b/haystack/components/preprocessors/markdown_header_splitter.py @@ -79,24 +79,10 @@ def __init__( self._header_split_levels_set = set(header_split_levels) self._header_pattern = re.compile(r"(?m)^(#{1,6}) (.+)$") # ATX-style .md-headers - # Matches fenced code blocks delimited by triple backticks (```) or triple tildes (~~~). - # Broken down: - # ^ - fence must start at the beginning of a line (MULTILINE) - # (?P`{3,}|~{3,}) - # - named capture group "fence": three or more backticks OR three or - # more tildes. Capturing it allows the closing fence to be matched - # with a backreference, so ```-opened blocks must close with ``` - # and ~~~-opened blocks must close with ~~~. - # [^\n]* - optional language identifier (e.g. "python") and any other text - # on the opening fence line, up to the newline - # \n - newline ending the opening fence line - # .*? - the code block body, matched lazily (DOTALL so . matches newlines) - # ^(?P=fence) - closing fence: must be identical to the opening fence (backreference), - # and must start at the beginning of a line - # \s*$ - optional trailing whitespace after the closing fence - self._code_block_pattern = re.compile( - r"^(?P`{3,}|~{3,})[^\n]*\n.*?^(?P=fence)\s*$", re.MULTILINE | re.DOTALL - ) + # CommonMark allows up to three spaces before an opening fence. A backtick fence's info + # string cannot itself contain a backtick; that condition is checked while scanning. + self._code_block_open_pattern = re.compile(r" {0,3}(?P`{3,}|~{3,})(?P[^\r\n]*)") + self._code_block_close_pattern = re.compile(r" {0,3}(?P`{3,}|~{3,})[ \t]*") self._is_warmed_up = False @@ -119,7 +105,39 @@ def warm_up(self) -> None: def _code_block_spans(self, text: str) -> list[tuple[int, int]]: """Return the (start, end) character spans of all fenced code blocks in text.""" - return [(m.start(), m.end()) for m in self._code_block_pattern.finditer(text)] + spans: list[tuple[int, int]] = [] + block_start: int | None = None + opening_character = "" + opening_length = 0 + offset = 0 + + for line in text.splitlines(keepends=True): + line_without_ending = line.rstrip("\r\n") + + if block_start is None: + match = self._code_block_open_pattern.fullmatch(line_without_ending) + if match: + fence = match.group("fence") + info = match.group("info") + if fence[0] != "`" or "`" not in info: + block_start = offset + opening_character = fence[0] + opening_length = len(fence) + else: + match = self._code_block_close_pattern.fullmatch(line_without_ending) + if match: + fence = match.group("fence") + if fence[0] == opening_character and len(fence) >= opening_length: + spans.append((block_start, offset + len(line))) + block_start = None + + offset += len(line) + + # An unclosed fenced block continues to the end of its containing document. + if block_start is not None: + spans.append((block_start, len(text))) + + return spans def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: """Split text by ATX-style headers (#) and create chunks with appropriate metadata.""" diff --git a/releasenotes/notes/markdown-fence-boundaries-20f1dacc1efec6c8.yaml b/releasenotes/notes/markdown-fence-boundaries-20f1dacc1efec6c8.yaml new file mode 100644 index 0000000000..ff54384340 --- /dev/null +++ b/releasenotes/notes/markdown-fence-boundaries-20f1dacc1efec6c8.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixed ``MarkdownHeaderSplitter`` treating hash-prefixed lines inside indented, unclosed, or + longer-delimiter fenced code blocks as Markdown headers. diff --git a/test/components/preprocessors/test_markdown_header_splitter.py b/test/components/preprocessors/test_markdown_header_splitter.py index 686c3e5b0c..18beaee033 100644 --- a/test/components/preprocessors/test_markdown_header_splitter.py +++ b/test/components/preprocessors/test_markdown_header_splitter.py @@ -508,6 +508,45 @@ def test_longer_fence_delimiters(self): assert docs[1].meta["header"] == "Real Subheader" assert "not a header" not in [doc.meta["header"] for doc in docs] + @pytest.mark.parametrize(("opening", "closing"), [("```python", "````"), ("~~~bash", "~~~~")]) + def test_closing_fence_can_be_longer_than_opening(self, opening, closing): + """A closing fence can contain more markers than its opening fence.""" + text = f"# Real Header\n{opening}\n# not a header\n{closing}\n## Real Subheader\nContent.\n" + + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + + assert [doc.meta["header"] for doc in docs] == ["Real Header", "Real Subheader"] + assert "".join(doc.content for doc in docs) == text + + @pytest.mark.parametrize("indent", [" ", " ", " "]) + def test_fences_can_be_indented(self, indent): + """Opening and closing fences can be indented by up to three spaces.""" + text = f"# Real Header\n{indent}```python\n# not a header\n{indent}```\n## Real Subheader\nContent.\n" + + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + + assert [doc.meta["header"] for doc in docs] == ["Real Header", "Real Subheader"] + assert "".join(doc.content for doc in docs) == text + + def test_unclosed_fence_continues_to_end_of_document(self): + """An unclosed fence prevents hash-prefixed code lines from becoming headers.""" + text = "# Real Header\n```python\n# not a header\n## also not a header\n" + + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + + assert len(docs) == 1 + assert docs[0].meta["header"] == "Real Header" + assert docs[0].content == text + + def test_shorter_fence_does_not_close_block(self): + """A closing fence must contain at least as many markers as its opening fence.""" + text = "# Real Header\n````python\n```\n# not a header\n````\n## Real Subheader\nContent.\n" + + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + + assert [doc.meta["header"] for doc in docs] == ["Real Header", "Real Subheader"] + assert "".join(doc.content for doc in docs) == text + def test_code_block_with_no_real_headers(self): """If the only hash lines are inside code blocks, the document is returned unchunked.""" text = "Plain text before code.\n```\n# entirely fake\n```\nPlain text after code.\n"