diff --git a/pageindex/page_index_md.py b/pageindex/page_index_md.py index 86ef2a145..1e7fb6cfa 100644 --- a/pageindex/page_index_md.py +++ b/pageindex/page_index_md.py @@ -32,26 +32,34 @@ async def generate_summaries_for_structure_md(structure, summary_token_threshold def extract_nodes_from_markdown(markdown_content): header_pattern = r'^(#{1,6})\s+(.+)$' bold_heading_pattern = r'^\*\*(.+?)\*\*\s*$' - code_block_pattern = r'^```' + code_block_pattern = r'^ {0,3}(`{3,}|~{3,})(.*)$' node_list = [] lines = markdown_content.split('\n') - in_code_block = False + fence = None for line_num, line in enumerate(lines, 1): stripped_line = line.strip() - # Check for code block delimiters (triple backticks) - if re.match(code_block_pattern, stripped_line): - in_code_block = not in_code_block + delimiter = re.match(code_block_pattern, line) + if fence is not None: + if (delimiter and delimiter.group(1)[0] == fence[0] + and len(delimiter.group(1)) >= len(fence) + and not delimiter.group(2).strip()): + fence = None continue - + if delimiter: + marker, info = delimiter.groups() + if marker[0] != '`' or '`' not in info: + fence = marker + continue + # Skip empty lines if not stripped_line: continue # Only look for headers when not inside a code block - if not in_code_block: + if fence is None: match = re.match(header_pattern, stripped_line) if match: title = match.group(2).strip() diff --git a/tests/test_page_index_md.py b/tests/test_page_index_md.py index 0f4581090..be2e4188f 100644 --- a/tests/test_page_index_md.py +++ b/tests/test_page_index_md.py @@ -4,6 +4,24 @@ class ExtractNodesFromMarkdownTest(unittest.TestCase): + def test_fenced_examples_do_not_become_document_headings(self): + cases = [ + "~~~python\n# example\n**also code**\n~~~", + "````markdown\n```python\n# example\n```\n````", + "```python\n~~~\n# example\n~~~\n```", + "```python\n``` not a closing fence\n# example\n```", + ] + for code in cases: + with self.subTest(code=code): + text = "# Before\n" + code + "\n## After" + nodes, lines = extract_nodes_from_markdown(text) + self.assertEqual([n["node_title"] for n in nodes], ["Before", "After"]) + self.assertEqual(nodes[-1]["line_num"], len(lines)) + + def test_unclosed_tilde_fence_hides_remaining_headings(self): + nodes, _ = extract_nodes_from_markdown("# Before\n~~~\n## Example") + self.assertEqual([n["node_title"] for n in nodes], ["Before"]) + def test_skips_bold_heading_with_only_whitespace(self): nodes, _ = extract_nodes_from_markdown("** **\n**Valid heading**")