diff --git a/app/scripts/sync_sources.py b/app/scripts/sync_sources.py index 6c9096fd..996a3a65 100755 --- a/app/scripts/sync_sources.py +++ b/app/scripts/sync_sources.py @@ -20,6 +20,7 @@ import sys import time import urllib.error +import urllib.parse import urllib.request from dataclasses import asdict, dataclass from datetime import datetime, timezone @@ -51,7 +52,7 @@ # Bump when HTMLContentExtractor, clean_bclaws_content, or _strip_bclaws_chrome # changes the Markdown those functions emit. A new value forces a full fetch. -EXTRACTOR_VERSION = "1" +EXTRACTOR_VERSION = "2" # A fetch that fails before any HTTP status arrives is retried with backoff. # An HTTP status response (including 4xx and 5xx) is not retried. @@ -182,6 +183,14 @@ def dump(data: dict[str, Any]) -> str: return yaml.safe_dump(data, sort_keys=False) +def _heading_slug(text: str) -> str: + """GitHub-style heading slug: lowercase, drop punctuation, spaces to hyphens.""" + text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text) + text = re.sub(r"\s+", " ", text.replace("*", "")).strip().lower() + text = re.sub(r"[^\w\- ]", "", text) + return text.replace(" ", "-") + + class _OpeningMarker(str): """An emphasis marker that opens a span, so cell text can keep whitespace outside it.""" @@ -409,6 +418,12 @@ def __init__(self, target_selector: str | None = None): self.is_italic = False self.link_text_tokens: list[str] = [] self.tables: list[_TableState] = [] + # In-page anchors: an id or name on a heading, or on an element inside it, + # maps to that heading's Markdown slug so "#id" links still resolve. + self.heading_start: int | None = None + self.heading_ids: list[str] = [] + self.anchor_slugs: dict[str, str] = {} + self.slug_counts: dict[str, int] = {} def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag_lower = tag.lower() @@ -444,6 +459,12 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None if not self.inside_target: return + is_heading = re.match(r"^h[1-6]$", tag_lower) is not None + if is_heading or self.heading_level: + for key in ("id", "name"): + if attr_dict.get(key): + self.heading_ids.append(attr_dict[key]) + if tag_lower == "table": self._open_table(attr_dict) elif tag_lower == "tr" and self._in_data_table(): @@ -454,10 +475,12 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None elif re.match(r"^h[1-6]$", tag_lower): self.heading_level = int(tag_lower[1]) self.tokens.append(f"\n\n{'#' * self.heading_level} ") + self.heading_start = len(self.tokens) elif tag_lower in self.BLOCK_TAGS: self.tokens.append("\n\n") elif tag_lower == "br": - self.tokens.append(_LineBreak("\n")) + # A Markdown heading ends at the newline, so keep a heading on one line. + self.tokens.append(" " if self.heading_start is not None else _LineBreak("\n")) elif tag_lower == "li": self.tokens.append("\n- ") elif tag_lower in ("strong", "b"): @@ -608,6 +631,7 @@ def handle_endtag(self, tag: str) -> None: elif tag_lower in ("td", "th") and self._in_data_table(): self._close_cell() elif re.match(r"^h[1-6]$", tag_lower): + self._record_heading_anchor() self.heading_level = 0 self.tokens.append("\n\n") elif tag_lower in self.BLOCK_TAGS: @@ -671,9 +695,33 @@ def _matches_selector(self, tag: str, attrs: dict[str, str]) -> str | None: return selector return None + def _record_heading_anchor(self) -> None: + """Map the closing heading's ids to the slug a Markdown renderer gives it.""" + if self.heading_start is None: + return + text = "".join(self.tokens[self.heading_start:]) + self.heading_start = None + ids, self.heading_ids = self.heading_ids, [] + base = _heading_slug(text) + if not base: + return + slug = base + while slug in self.slug_counts: + self.slug_counts[base] += 1 + slug = f"{base}-{self.slug_counts[base]}" + self.slug_counts[slug] = 0 + for anchor_id in ids: + self.anchor_slugs.setdefault(anchor_id, slug) + def get_markdown(self) -> str: self._close_open_tables() raw_text = "".join(self.tokens) + if self.anchor_slugs: + raw_text = re.sub( + r"\]\(#([^)\s]+)\)", + lambda m: f"](#{self.anchor_slugs.get(urllib.parse.unquote(m.group(1)), m.group(1))})", + raw_text, + ) # Clean extra spaces & lines lines = [] for line in raw_text.splitlines(): diff --git a/app/tests/test_sync_sources.py b/app/tests/test_sync_sources.py index 11574d84..b41fd62c 100644 --- a/app/tests/test_sync_sources.py +++ b/app/tests/test_sync_sources.py @@ -2033,7 +2033,7 @@ def test_check_extractor_version_change_does_not_match(monkeypatch, tmp_path): _target, digest = _write_doc(tmp_path) entry = _entry_with_validators(digest) entry.extraction_fingerprint = extraction_fingerprint(entry) - monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", "2") + monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", sync_sources.EXTRACTOR_VERSION + "-next") html = _html_paragraph("Kept body.") _title, body = extract_content(html, "html_selector", "#body") assert compute_content_hash(extract_substantive_body(body)) == digest @@ -2057,7 +2057,7 @@ def test_sync_extractor_version_change_does_not_skip_the_write(monkeypatch, tmp_ before = target.read_text(encoding="utf-8") entry = _entry_with_validators(digest) entry.extraction_fingerprint = extraction_fingerprint(entry) - monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", "2") + monkeypatch.setattr(sync_sources, "EXTRACTOR_VERSION", sync_sources.EXTRACTOR_VERSION + "-next") def responder(req: urllib.request.Request) -> _UrlResponse: _assert_unconditional(req) @@ -2502,3 +2502,82 @@ def fake_check(entry: SourceEntry, repo_root: Path) -> DriftResult: body = alert.read_text(encoding="utf-8") assert ("Sources That Could Not Be Reached" in body) == ("UNREACHABLE" in statuses) assert ("Sources That Failed To Check" in body) == ("ERROR" in statuses) + + +def test_in_page_anchor_links_point_at_heading_slugs(): + """An "#id" link to a heading's own id or a nested anchor resolves to the Markdown slug.""" + raw_html = """ +
+ +

Employee responsibility

+

Body.

+

Role of the Office

+

Section 2 — Scope, and limits

+

Employee responsibility

+
+ """ + extractor = HTMLContentExtractor(target_selector="#body") + extractor.feed(raw_html) + markdown = extractor.get_markdown() + + assert "[Employee responsibility](#employee-responsibility)" in markdown + assert "[Role of the Office](#role-of-the-office)" in markdown + assert "[Section two](#section-2--scope-and-limits)" in markdown + assert "[Again](#employee-responsibility-1)" in markdown + assert "[Missing](#nowhere)" in markdown + + +def test_in_page_anchor_slugs_stay_unique_and_fragments_decode(): + """A heading whose text matches an earlier suffixed slug still gets its own slug.""" + raw_html = """ +
+

A B C P

+

Foo

+

Foo 1

+

Foo

+

Part one

+
+ """ + extractor = HTMLContentExtractor(target_selector="#body") + extractor.feed(raw_html) + markdown = extractor.get_markdown() + + assert "[A](#foo)" in markdown + assert "[B](#foo-1)" in markdown + assert "[C](#foo-2)" in markdown + assert "[P](#part-one)" in markdown + + +def test_heading_line_break_stays_on_heading_line(): + """A
inside a heading must not split it, so the slug matches the rendered heading.""" + raw_html = """ +
+

Go

+

First
Second

+
+ """ + extractor = HTMLContentExtractor(target_selector="#body") + extractor.feed(raw_html) + markdown = extractor.get_markdown() + + assert "## First Second" in markdown + assert "[Go](#first-second)" in markdown + + +def test_bclaws_in_page_anchor_links_point_at_heading_slugs(): + """The bclaws cleaner shares the extractor, so its anchors resolve too.""" + raw_html = """ +
+

1

+

Definitions

+

Text.

+
+ """ + markdown = clean_bclaws_content(raw_html, selector="#contentsscroll") + assert "[1](#definitions)" in markdown