fix: handle CommonMark fenced code boundaries

haystack/components/preprocessors/markdown_header_splitter.py

@@ -79,24 +79,10 @@ def __init__(
         self._header_split_levels_set = set(header_split_levels)
         self._header_pattern = re.compile(r"(?m)^(#{1,6}) (.+)$")  # ATX-style .md-headers
 
-        # Matches fenced code blocks delimited by triple backticks (```) or triple tildes (~~~).
-        # Broken down:
-        #   ^                 - fence must start at the beginning of a line (MULTILINE)
-        #   (?P<fence>`{3,}|~{3,})
-        #                     - named capture group "fence": three or more backticks OR three or
-        #                       more tildes. Capturing it allows the closing fence to be matched
-        #                       with a backreference, so ```-opened blocks must close with ```
-        #                       and ~~~-opened blocks must close with ~~~.
-        #   [^\n]*            - optional language identifier (e.g. "python") and any other text
-        #                       on the opening fence line, up to the newline
-        #   \n                - newline ending the opening fence line
-        #   .*?               - the code block body, matched lazily (DOTALL so . matches newlines)
-        #   ^(?P=fence)       - closing fence: must be identical to the opening fence (backreference),
-        #                       and must start at the beginning of a line
-        #   \s*$              - optional trailing whitespace after the closing fence
-        self._code_block_pattern = re.compile(
-            r"^(?P<fence>`{3,}|~{3,})[^\n]*\n.*?^(?P=fence)\s*$", re.MULTILINE | re.DOTALL
-        )
+        # CommonMark allows up to three spaces before an opening fence. A backtick fence's info
+        # string cannot itself contain a backtick; that condition is checked while scanning.
+        self._code_block_open_pattern = re.compile(r" {0,3}(?P<fence>`{3,}|~{3,})(?P<info>[^\r\n]*)")
+        self._code_block_close_pattern = re.compile(r" {0,3}(?P<fence>`{3,}|~{3,})[ \t]*")
 
         self._is_warmed_up = False
 
@@ -119,7 +105,39 @@ def warm_up(self) -> None:
 
     def _code_block_spans(self, text: str) -> list[tuple[int, int]]:
         """Return the (start, end) character spans of all fenced code blocks in text."""
-        return [(m.start(), m.end()) for m in self._code_block_pattern.finditer(text)]
+        spans: list[tuple[int, int]] = []
+        block_start: int | None = None
+        opening_character = ""
+        opening_length = 0
+        offset = 0
+
+        for line in text.splitlines(keepends=True):
+            line_without_ending = line.rstrip("\r\n")
+
+            if block_start is None:
+                match = self._code_block_open_pattern.fullmatch(line_without_ending)
+                if match:
+                    fence = match.group("fence")
+                    info = match.group("info")
+                    if fence[0] != "`" or "`" not in info:
+                        block_start = offset
+                        opening_character = fence[0]
+                        opening_length = len(fence)
+            else:
+                match = self._code_block_close_pattern.fullmatch(line_without_ending)
+                if match:
+                    fence = match.group("fence")
+                    if fence[0] == opening_character and len(fence) >= opening_length:
+                        spans.append((block_start, offset + len(line)))
+                        block_start = None
+
+            offset += len(line)
+
+        # An unclosed fenced block continues to the end of its containing document.
+        if block_start is not None:
+            spans.append((block_start, len(text)))
+
+        return spans
 
     def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]:
         """Split text by ATX-style headers (#) and create chunks with appropriate metadata."""

releasenotes/notes/markdown-fence-boundaries-20f1dacc1efec6c8.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed ``MarkdownHeaderSplitter`` treating hash-prefixed lines inside indented, unclosed, or
+    longer-delimiter fenced code blocks as Markdown headers.

test/components/preprocessors/test_markdown_header_splitter.py

@@ -508,6 +508,45 @@ def test_longer_fence_delimiters(self):
         assert docs[1].meta["header"] == "Real Subheader"
         assert "not a header" not in [doc.meta["header"] for doc in docs]
 
+    @pytest.mark.parametrize(("opening", "closing"), [("```python", "````"), ("~~~bash", "~~~~")])
+    def test_closing_fence_can_be_longer_than_opening(self, opening, closing):
+        """A closing fence can contain more markers than its opening fence."""
+        text = f"# Real Header\n{opening}\n# not a header\n{closing}\n## Real Subheader\nContent.\n"
+
+        docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+
+        assert [doc.meta["header"] for doc in docs] == ["Real Header", "Real Subheader"]
+        assert "".join(doc.content for doc in docs) == text
+
+    @pytest.mark.parametrize("indent", [" ", "  ", "   "])
+    def test_fences_can_be_indented(self, indent):
+        """Opening and closing fences can be indented by up to three spaces."""
+        text = f"# Real Header\n{indent}```python\n# not a header\n{indent}```\n## Real Subheader\nContent.\n"
+
+        docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+
+        assert [doc.meta["header"] for doc in docs] == ["Real Header", "Real Subheader"]
+        assert "".join(doc.content for doc in docs) == text
+
+    def test_unclosed_fence_continues_to_end_of_document(self):
+        """An unclosed fence prevents hash-prefixed code lines from becoming headers."""
+        text = "# Real Header\n```python\n# not a header\n## also not a header\n"
+
+        docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+
+        assert len(docs) == 1
+        assert docs[0].meta["header"] == "Real Header"
+        assert docs[0].content == text
+
+    def test_shorter_fence_does_not_close_block(self):
+        """A closing fence must contain at least as many markers as its opening fence."""
+        text = "# Real Header\n````python\n```\n# not a header\n````\n## Real Subheader\nContent.\n"
+
+        docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+
+        assert [doc.meta["header"] for doc in docs] == ["Real Header", "Real Subheader"]
+        assert "".join(doc.content for doc in docs) == text
+
     def test_code_block_with_no_real_headers(self):
         """If the only hash lines are inside code blocks, the document is returned unchunked."""
         text = "Plain text before code.\n```\n# entirely fake\n```\nPlain text after code.\n"