From 6b059a2d6113cc233808719a5563f29ce9a98965 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Mon, 19 Oct 2026 11:12:45 -0700 Subject: [PATCH 1/3] fix(sync): point in-page anchor links at Markdown heading slugs --- app/scripts/sync_sources.py | 46 +++++++++++++++++++++++++++++++++- app/tests/test_sync_sources.py | 46 ++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/app/scripts/sync_sources.py b/app/scripts/sync_sources.py index 5465270d..471f6fc1 100755 --- a/app/scripts/sync_sources.py +++ b/app/scripts/sync_sources.py @@ -50,7 +50,7 @@ # Bump when HTMLContentExtractor, clean_bclaws_content, or _strip_bclaws_chrome # changes the Markdown those functions emit. A new value forces a full fetch. -EXTRACTOR_VERSION = "1" +EXTRACTOR_VERSION = "2" @dataclass @@ -169,6 +169,14 @@ def dump(data: dict[str, Any]) -> str: return yaml.safe_dump(data, sort_keys=False) +def _heading_slug(text: str) -> str: + """GitHub-style heading slug: lowercase, drop punctuation, spaces to hyphens.""" + text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) + text = re.sub(r"\s+", " ", text.replace("*", "")).strip().lower() + text = re.sub(r"[^\w\- ]", "", text) + return text.replace(" ", "-") + + class _OpeningMarker(str): """An emphasis marker that opens a span, so cell text can keep whitespace outside it.""" @@ -396,6 +404,12 @@ def __init__(self, target_selector: str | None = None): self.is_italic = False self.link_text_tokens: list[str] = [] self.tables: list[_TableState] = [] + # In-page anchors: an id or name on a heading, or on an element inside it, + # maps to that heading's Markdown slug so "#id" links still resolve. + self.heading_start: int | None = None + self.heading_ids: list[str] = [] + self.anchor_slugs: dict[str, str] = {} + self.slug_counts: dict[str, int] = {} def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag_lower = tag.lower() @@ -431,6 +445,12 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None if not self.inside_target: return + is_heading = re.match(r"^h[1-6]$", tag_lower) is not None + if is_heading or self.heading_level: + for key in ("id", "name"): + if attr_dict.get(key): + self.heading_ids.append(attr_dict[key]) + if tag_lower == "table": self._open_table(attr_dict) elif tag_lower == "tr" and self._in_data_table(): @@ -441,6 +461,7 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None elif re.match(r"^h[1-6]$", tag_lower): self.heading_level = int(tag_lower[1]) self.tokens.append(f"\n\n{'#' * self.heading_level} ") + self.heading_start = len(self.tokens) elif tag_lower in self.BLOCK_TAGS: self.tokens.append("\n\n") elif tag_lower == "br": @@ -595,6 +616,7 @@ def handle_endtag(self, tag: str) -> None: elif tag_lower in ("td", "th") and self._in_data_table(): self._close_cell() elif re.match(r"^h[1-6]$", tag_lower): + self._record_heading_anchor() self.heading_level = 0 self.tokens.append("\n\n") elif tag_lower in self.BLOCK_TAGS: @@ -658,9 +680,31 @@ def _matches_selector(self, tag: str, attrs: dict[str, str]) -> str | None: return selector return None + def _record_heading_anchor(self) -> None: + """Map the closing heading's ids to the slug a Markdown renderer gives it.""" + if self.heading_start is None: + return + text = "".join(self.tokens[self.heading_start:]) + self.heading_start = None + ids, self.heading_ids = self.heading_ids, [] + base = _heading_slug(text) + if not base: + return + count = self.slug_counts.get(base, 0) + self.slug_counts[base] = count + 1 + slug = base if count == 0 else f"{base}-{count}" + for anchor_id in ids: + self.anchor_slugs.setdefault(anchor_id, slug) + def get_markdown(self) -> str: self._close_open_tables() raw_text = "".join(self.tokens) + if self.anchor_slugs: + raw_text = re.sub( + r"\]\(#([^)\s]+)\)", + lambda m: f"](#{self.anchor_slugs.get(m.group(1), m.group(1))})", + raw_text, + ) # Clean extra spaces & lines lines = [] for line in raw_text.splitlines(): diff --git a/app/tests/test_sync_sources.py b/app/tests/test_sync_sources.py index ba64b628..76625a02 100644 --- a/app/tests/test_sync_sources.py +++ b/app/tests/test_sync_sources.py @@ -2033,7 +2033,7 @@ def test_check_extractor_version_change_does_not_match(monkeypatch, tmp_path): _target, digest = _write_doc(tmp_path) entry = _entry_with_validators(digest) entry.extraction_fingerprint = extraction_fingerprint(entry) - monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", "2") + monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", sync_sources.EXTRACTOR_VERSION + "-next") html = _html_paragraph("Kept body.") _title, body = extract_content(html, "html_selector", "#body") assert compute_content_hash(extract_substantive_body(body)) == digest @@ -2057,7 +2057,7 @@ def test_sync_extractor_version_change_does_not_skip_the_write(monkeypatch, tmp_ before = target.read_text(encoding="utf-8") entry = _entry_with_validators(digest) entry.extraction_fingerprint = extraction_fingerprint(entry) - monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", "2") + monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", sync_sources.EXTRACTOR_VERSION + "-next") def responder(req: urllib.request.Request) -> _UrlResponse: _assert_unconditional(req) @@ -2372,3 +2372,45 @@ def test_sync_regenerates_manifest_and_drawer(mock_manifest, tmp_path, monkeypat mock_manifest.assert_called_once_with(data_dir=tmp_path / "app" / "data") + + +def test_in_page_anchor_links_point_at_heading_slugs(): + """An "#id" link to a heading's own id or a nested anchor resolves to the Markdown slug.""" + raw_html = """ +
+ +

Employee responsibility

+

Body.

+

Role of the Office

+

Section 2 — Scope, and limits

+

Employee responsibility

+
+ """ + extractor = HTMLContentExtractor(target_selector="#body") + extractor.feed(raw_html) + markdown = extractor.get_markdown() + + assert "[Employee responsibility](#employee-responsibility)" in markdown + assert "[Role of the Office](#role-of-the-office)" in markdown + assert "[Section two](#section-2--scope-and-limits)" in markdown + assert "[Again](#employee-responsibility-1)" in markdown + assert "[Missing](#nowhere)" in markdown + + +def test_bclaws_in_page_anchor_links_point_at_heading_slugs(): + """The bclaws cleaner shares the extractor, so its anchors resolve too.""" + raw_html = """ +
+

1

+

Definitions

+

Text.

+
+ """ + markdown = clean_bclaws_content(raw_html, selector="#contentsscroll") + assert "[1](#definitions)" in markdown From 9377f2149f305295c29f013c68386ce539782bea Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 20 Oct 2026 14:03:53 -0700 Subject: [PATCH 2/3] fix(sync): keep heading slugs unique and decode link fragments --- app/scripts/sync_sources.py | 11 +++++++---- app/tests/test_sync_sources.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/app/scripts/sync_sources.py b/app/scripts/sync_sources.py index 8484b3e4..ded9e008 100755 --- a/app/scripts/sync_sources.py +++ b/app/scripts/sync_sources.py @@ -20,6 +20,7 @@ import sys import time import urllib.error +import urllib.parse import urllib.request from dataclasses import asdict, dataclass from datetime import datetime, timezone @@ -703,9 +704,11 @@ def _record_heading_anchor(self) -> None: base = _heading_slug(text) if not base: return - count = self.slug_counts.get(base, 0) - self.slug_counts[base] = count + 1 - slug = base if count == 0 else f"{base}-{count}" + slug = base + while slug in self.slug_counts: + self.slug_counts[base] += 1 + slug = f"{base}-{self.slug_counts[base]}" + self.slug_counts[slug] = 0 for anchor_id in ids: self.anchor_slugs.setdefault(anchor_id, slug) @@ -715,7 +718,7 @@ def get_markdown(self) -> str: if self.anchor_slugs: raw_text = re.sub( r"\]\(#([^)\s]+)\)", - lambda m: f"](#{self.anchor_slugs.get(m.group(1), m.group(1))})", + lambda m: f"](#{self.anchor_slugs.get(urllib.parse.unquote(m.group(1)), m.group(1))})", raw_text, ) # Clean extra spaces & lines diff --git a/app/tests/test_sync_sources.py b/app/tests/test_sync_sources.py index 5be077ed..1ebb7939 100644 --- a/app/tests/test_sync_sources.py +++ b/app/tests/test_sync_sources.py @@ -2533,6 +2533,27 @@ def test_in_page_anchor_links_point_at_heading_slugs(): assert "[Missing](#nowhere)" in markdown +def test_in_page_anchor_slugs_stay_unique_and_fragments_decode(): + """A heading whose text matches an earlier suffixed slug still gets its own slug.""" + raw_html = """ +
+

A B C P

+

Foo

+

Foo 1

+

Foo

+

Part one

+
+ """ + extractor = HTMLContentExtractor(target_selector="#body") + extractor.feed(raw_html) + markdown = extractor.get_markdown() + + assert "[A](#foo)" in markdown + assert "[B](#foo-1)" in markdown + assert "[C](#foo-2)" in markdown + assert "[P](#part-one)" in markdown + + def test_bclaws_in_page_anchor_links_point_at_heading_slugs(): """The bclaws cleaner shares the extractor, so its anchors resolve too.""" raw_html = """ From 14de5da156ac9d639c09b0738dd36de57dc6e5e3 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Tue, 20 Oct 2026 14:11:57 -0700 Subject: [PATCH 3/3] fix(sync): keep headings with line breaks on one line --- app/scripts/sync_sources.py | 3 ++- app/tests/test_sync_sources.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/app/scripts/sync_sources.py b/app/scripts/sync_sources.py index ded9e008..996a3a65 100755 --- a/app/scripts/sync_sources.py +++ b/app/scripts/sync_sources.py @@ -479,7 +479,8 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None elif tag_lower in self.BLOCK_TAGS: self.tokens.append("\n\n") elif tag_lower == "br": - self.tokens.append(_LineBreak("\n")) + # A Markdown heading ends at the newline, so keep a heading on one line. + self.tokens.append(" " if self.heading_start is not None else _LineBreak("\n")) elif tag_lower == "li": self.tokens.append("\n- ") elif tag_lower in ("strong", "b"): diff --git a/app/tests/test_sync_sources.py b/app/tests/test_sync_sources.py index 1ebb7939..b41fd62c 100644 --- a/app/tests/test_sync_sources.py +++ b/app/tests/test_sync_sources.py @@ -2554,6 +2554,22 @@ def test_in_page_anchor_slugs_stay_unique_and_fragments_decode(): assert "[P](#part-one)" in markdown +def test_heading_line_break_stays_on_heading_line(): + """A
inside a heading must not split it, so the slug matches the rendered heading.""" + raw_html = """ +
+

Go

+

First
Second

+
+ """ + extractor = HTMLContentExtractor(target_selector="#body") + extractor.feed(raw_html) + markdown = extractor.get_markdown() + + assert "## First Second" in markdown + assert "[Go](#first-second)" in markdown + + def test_bclaws_in_page_anchor_links_point_at_heading_slugs(): """The bclaws cleaner shares the extractor, so its anchors resolve too.""" raw_html = """