fix(sync): point in-page anchor links at Markdown heading slugs

app/scripts/sync_sources.py

@@ -20,6 +20,7 @@
 import sys
 import time
 import urllib.error
+import urllib.parse
 import urllib.request
 from dataclasses import asdict, dataclass
 from datetime import datetime, timezone
@@ -51,7 +52,7 @@
 
 # Bump when HTMLContentExtractor, clean_bclaws_content, or _strip_bclaws_chrome
 # changes the Markdown those functions emit. A new value forces a full fetch.
-EXTRACTOR_VERSION = "1"
+EXTRACTOR_VERSION = "2"
 
 # A fetch that fails before any HTTP status arrives is retried with backoff.
 # An HTTP status response (including 4xx and 5xx) is not retried.
@@ -182,6 +183,14 @@ def dump(data: dict[str, Any]) -> str:
         return yaml.safe_dump(data, sort_keys=False)
 
 
+def _heading_slug(text: str) -> str:
+    """GitHub-style heading slug: lowercase, drop punctuation, spaces to hyphens."""
+    text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
+    text = re.sub(r"\s+", " ", text.replace("*", "")).strip().lower()
+    text = re.sub(r"[^\w\- ]", "", text)
+    return text.replace(" ", "-")
+
+
 class _OpeningMarker(str):
     """An emphasis marker that opens a span, so cell text can keep whitespace outside it."""
 
@@ -409,6 +418,12 @@ def __init__(self, target_selector: str | None = None):
         self.is_italic = False
         self.link_text_tokens: list[str] = []
         self.tables: list[_TableState] = []
+        # In-page anchors: an id or name on a heading, or on an element inside it,
+        # maps to that heading's Markdown slug so "#id" links still resolve.
+        self.heading_start: int | None = None
+        self.heading_ids: list[str] = []
+        self.anchor_slugs: dict[str, str] = {}
+        self.slug_counts: dict[str, int] = {}
 
     def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
         tag_lower = tag.lower()
@@ -444,6 +459,12 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
         if not self.inside_target:
             return
 
+        is_heading = re.match(r"^h[1-6]$", tag_lower) is not None
+        if is_heading or self.heading_level:
+            for key in ("id", "name"):
+                if attr_dict.get(key):
+                    self.heading_ids.append(attr_dict[key])
+
         if tag_lower == "table":
             self._open_table(attr_dict)
         elif tag_lower == "tr" and self._in_data_table():
@@ -454,10 +475,12 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
         elif re.match(r"^h[1-6]$", tag_lower):
             self.heading_level = int(tag_lower[1])
             self.tokens.append(f"\n\n{'#' * self.heading_level} ")
+            self.heading_start = len(self.tokens)
         elif tag_lower in self.BLOCK_TAGS:
             self.tokens.append("\n\n")
         elif tag_lower == "br":
-            self.tokens.append(_LineBreak("\n"))
+            # A Markdown heading ends at the newline, so keep a heading on one line.
+            self.tokens.append(" " if self.heading_start is not None else _LineBreak("\n"))
         elif tag_lower == "li":
             self.tokens.append("\n- ")
         elif tag_lower in ("strong", "b"):
@@ -608,6 +631,7 @@ def handle_endtag(self, tag: str) -> None:
         elif tag_lower in ("td", "th") and self._in_data_table():
             self._close_cell()
         elif re.match(r"^h[1-6]$", tag_lower):
+            self._record_heading_anchor()
             self.heading_level = 0
             self.tokens.append("\n\n")
         elif tag_lower in self.BLOCK_TAGS:
@@ -671,9 +695,33 @@ def _matches_selector(self, tag: str, attrs: dict[str, str]) -> str | None:
                 return selector
         return None
 
+    def _record_heading_anchor(self) -> None:
+        """Map the closing heading's ids to the slug a Markdown renderer gives it."""
+        if self.heading_start is None:
+            return
+        text = "".join(self.tokens[self.heading_start:])
+        self.heading_start = None
+        ids, self.heading_ids = self.heading_ids, []
+        base = _heading_slug(text)
+        if not base:
+            return
+        slug = base
+        while slug in self.slug_counts:
+            self.slug_counts[base] += 1
+            slug = f"{base}-{self.slug_counts[base]}"
+        self.slug_counts[slug] = 0
+        for anchor_id in ids:
+            self.anchor_slugs.setdefault(anchor_id, slug)
+
     def get_markdown(self) -> str:
         self._close_open_tables()
         raw_text = "".join(self.tokens)
+        if self.anchor_slugs:
+            raw_text = re.sub(
+                r"\]\(#([^)\s]+)\)",
+                lambda m: f"](#{self.anchor_slugs.get(urllib.parse.unquote(m.group(1)), m.group(1))})",
+                raw_text,
+            )
         # Clean extra spaces & lines
         lines = []
         for line in raw_text.splitlines():

app/tests/test_sync_sources.py

@@ -2033,7 +2033,7 @@ def test_check_extractor_version_change_does_not_match(monkeypatch, tmp_path):
     _target, digest = _write_doc(tmp_path)
     entry = _entry_with_validators(digest)
     entry.extraction_fingerprint = extraction_fingerprint(entry)
-    monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", "2")
+    monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", sync_sources.EXTRACTOR_VERSION + "-next")
     html = _html_paragraph("Kept body.")
     _title, body = extract_content(html, "html_selector", "#body")
     assert compute_content_hash(extract_substantive_body(body)) == digest
@@ -2057,7 +2057,7 @@ def test_sync_extractor_version_change_does_not_skip_the_write(monkeypatch, tmp_
     before = target.read_text(encoding="utf-8")
     entry = _entry_with_validators(digest)
     entry.extraction_fingerprint = extraction_fingerprint(entry)
-    monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", "2")
+    monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", sync_sources.EXTRACTOR_VERSION + "-next")
 
     def responder(req: urllib.request.Request) -> _UrlResponse:
         _assert_unconditional(req)
@@ -2502,3 +2502,82 @@ def fake_check(entry: SourceEntry, repo_root: Path) -> DriftResult:
         body = alert.read_text(encoding="utf-8")
         assert ("Sources That Could Not Be Reached" in body) == ("UNREACHABLE" in statuses)
         assert ("Sources That Failed To Check" in body) == ("ERROR" in statuses)
+
+
+def test_in_page_anchor_links_point_at_heading_slugs():
+    """An "#id" link to a heading's own id or a nested anchor resolves to the Markdown slug."""
+    raw_html = """
+    <html><body><div id="body">
+      <ul>
+        <li><a href="#employee">Employee responsibility</a></li>
+        <li><a href="#roleof">Role of the Office</a></li>
+        <li><a href="#sec2">Section two</a></li>
+        <li><a href="#again">Again</a></li>
+        <li><a href="#nowhere">Missing</a></li>
+      </ul>
+      <h2><span><a id="employee" name="employee"></a>Employee responsibility</span></h2>
+      <p>Body.</p>
+      <h2><a name="roleof"></a>Role of the <strong>Office</strong></h2>
+      <h3 id="sec2">Section 2 &mdash; Scope, and limits</h3>
+      <h2 id="again">Employee responsibility</h2>
+    </div></body></html>
+    """
+    extractor = HTMLContentExtractor(target_selector="#body")
+    extractor.feed(raw_html)
+    markdown = extractor.get_markdown()
+
+    assert "[Employee responsibility](#employee-responsibility)" in markdown
+    assert "[Role of the Office](#role-of-the-office)" in markdown
+    assert "[Section two](#section-2--scope-and-limits)" in markdown
+    assert "[Again](#employee-responsibility-1)" in markdown
+    assert "[Missing](#nowhere)" in markdown
+
+
+def test_in_page_anchor_slugs_stay_unique_and_fragments_decode():
+    """A heading whose text matches an earlier suffixed slug still gets its own slug."""
+    raw_html = """
+    <html><body><div id="body">
+      <p><a href="#a">A</a> <a href="#b">B</a> <a href="#c">C</a> <a href="#part%3A1">P</a></p>
+      <h2 id="a">Foo</h2>
+      <h2 id="b">Foo 1</h2>
+      <h2 id="c">Foo</h2>
+      <h2 id="part:1">Part one</h2>
+    </div></body></html>
+    """
+    extractor = HTMLContentExtractor(target_selector="#body")
+    extractor.feed(raw_html)
+    markdown = extractor.get_markdown()
+
+    assert "[A](#foo)" in markdown
+    assert "[B](#foo-1)" in markdown
+    assert "[C](#foo-2)" in markdown
+    assert "[P](#part-one)" in markdown
+
+
+def test_heading_line_break_stays_on_heading_line():
+    """A <br> inside a heading must not split it, so the slug matches the rendered heading."""
+    raw_html = """
+    <html><body><div id="body">
+      <p><a href="#part">Go</a></p>
+      <h2 id="part">First<br>Second</h2>
+    </div></body></html>
+    """
+    extractor = HTMLContentExtractor(target_selector="#body")
+    extractor.feed(raw_html)
+    markdown = extractor.get_markdown()
+
+    assert "## First Second" in markdown
+    assert "[Go](#first-second)" in markdown
+
+
+def test_bclaws_in_page_anchor_links_point_at_heading_slugs():
+    """The bclaws cleaner shares the extractor, so its anchors resolve too."""
+    raw_html = """
+    <html><body><div id="contentsscroll">
+      <p><a href="#section1">1</a></p>
+      <h4 id="section1">Definitions</h4>
+      <p>Text.</p>
+    </div></body></html>
+    """
+    markdown = clean_bclaws_content(raw_html, selector="#contentsscroll")
+    assert "[1](#definitions)" in markdown