From 1f562b2f1a891057894918b781c848046ba4874d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 18:31:44 +0000 Subject: [PATCH] Resolve relative links and images against the feed's URL Since 0.2.0 the feed is downloaded with requests and its bytes are passed to feedparser.parse(), so feedparser no longer knows where the document came from. Relative link, image and values in summaries stayed relative (e.g. /news/1), which a dashboard then resolves against the Home Assistant host instead of the feed's site. fetch_feed now returns the content together with the final URL after redirects, and the new parse_feed() hands that URL to feedparser as Content-Location, which is what it resolved against in 0.1.12. The config flow validation uses the same helper. Fixes #183 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FvqVzfJq7yZSkZfSyG3YHT --- custom_components/feedparser/config_flow.py | 5 +- custom_components/feedparser/fetch.py | 19 ++++++- custom_components/feedparser/reader.py | 19 ++++++- tests/test_config_flow.py | 4 +- tests/test_hardening.py | 62 +++++++++++++++++++++ tests/test_proxy.py | 12 ++-- 6 files changed, 104 insertions(+), 17 deletions(-) diff --git a/custom_components/feedparser/config_flow.py b/custom_components/feedparser/config_flow.py index 9088b97f..b6f33349 100644 --- a/custom_components/feedparser/config_flow.py +++ b/custom_components/feedparser/config_flow.py @@ -7,7 +7,6 @@ from datetime import timedelta from typing import Any, cast -import feedparser # type: ignore[import] import requests from homeassistant.config_entries import ( SOURCE_IMPORT, @@ -46,7 +45,7 @@ ENTRY_VERSION, ) from .fetch import USER_AGENT, fetch_feed, is_valid_proxy -from .reader import compile_filter +from .reader import compile_filter, parse_feed from .source import ( URL_ENTITY_DOMAINS, InvalidFeedUrlError, @@ -626,7 +625,7 @@ def _validate_feed_url( Uses the same request (User-Agent, proxy, timeouts) as the sensor so a feed that passes here also works at runtime. """ - parsed = feedparser.parse(fetch_feed(feed_url, user_agent, proxy)) + parsed = parse_feed(fetch_feed(feed_url, user_agent, proxy)) # A real feed exposes a version (rss20, atom10, ...) even when it currently # has no entries; an HTML page or random document does not. if not parsed.get("version") and not parsed.get("entries"): diff --git a/custom_components/feedparser/fetch.py b/custom_components/feedparser/fetch.py index b5042c8a..2097398d 100644 --- a/custom_components/feedparser/fetch.py +++ b/custom_components/feedparser/fetch.py @@ -2,6 +2,7 @@ from __future__ import annotations +from typing import NamedTuple from urllib.parse import urlsplit import requests @@ -20,6 +21,14 @@ PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h") +class FetchedFeed(NamedTuple): + """A downloaded feed and the URL it was finally served from.""" + + content: bytes + # after redirects; feedparser resolves the feed's relative links against it + url: str + + def is_valid_proxy(proxy: str) -> bool: """Return True if `proxy` looks like scheme://[user:pass@]host:port.""" try: @@ -57,8 +66,12 @@ def redact_text(text: str, proxy: str) -> str: return text.replace(proxy, redact_proxy(proxy)) -def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes: - """Download a feed and return its raw bytes. +def fetch_feed( + url: str, + user_agent: str = USER_AGENT, + proxy: str = "", +) -> FetchedFeed: + """Download a feed and return its raw bytes and final URL. Raw bytes (not ``Response.text``) are returned so that feedparser can detect the document encoding from the XML prolog instead of relying on requests' guess. @@ -74,4 +87,4 @@ def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes proxies = {"http": proxy, "https": proxy} if proxy else None response = session.get(url, timeout=REQUEST_TIMEOUT, proxies=proxies) response.raise_for_status() - return response.content + return FetchedFeed(response.content, response.url or url) diff --git a/custom_components/feedparser/reader.py b/custom_components/feedparser/reader.py index e6082a33..6ada6062 100644 --- a/custom_components/feedparser/reader.py +++ b/custom_components/feedparser/reader.py @@ -15,13 +15,13 @@ import warnings from collections.abc import Mapping from datetime import UTC, datetime, timezone -from typing import Protocol, cast +from typing import Any, Protocol, cast import feedparser # type: ignore[import] from dateutil import parser from homeassistant.util import dt -from .fetch import USER_AGENT, fetch_feed, redact_proxy +from .fetch import USER_AGENT, FetchedFeed, fetch_feed, redact_proxy _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -33,6 +33,19 @@ DATE_KEYS = ("published", "updated", "created", "expired") +def parse_feed(fetched: FetchedFeed) -> Any: # noqa: ANN401 - feedparser is untyped + """Parse a downloaded feed, resolving relative URLs against where it came from. + + feedparser only knows the document's location when it downloads the feed + itself. Given bytes, it needs the URL as ``Content-Location`` to make + relative links, images and ```` in summaries absolute. + """ + return feedparser.parse( + fetched.content, + response_headers={"content-location": fetched.url}, + ) + + # Entry fields searched by the `match` / `exclude_match` filters. RSS # is exposed by feedparser as `summary`. FILTER_KEYS = ("title", "summary") @@ -157,7 +170,7 @@ def read(self: FeedReader, url: str | None = None) -> list[dict[str, object]]: _LOGGER.debug("Feed %s: Polling feed data from %s", self.name, url) parsed_feed = cast( "ParsedFeed", - feedparser.parse(fetch_feed(url, self._user_agent, self.proxy)), + parse_feed(fetch_feed(url, self._user_agent, self.proxy)), ) if not parsed_feed.entries: _LOGGER.warning("Feed %s: No data received.", self.name) diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py index e8468661..6064d627 100644 --- a/tests/test_config_flow.py +++ b/tests/test_config_flow.py @@ -16,7 +16,7 @@ _user_agent_from_input, ) from custom_components.feedparser.const import CONF_SCAN_INTERVAL, CONF_USER_AGENT -from custom_components.feedparser.fetch import USER_AGENT +from custom_components.feedparser.fetch import USER_AGENT, FetchedFeed def test_options_flow_reloads_config_entry() -> None: @@ -89,7 +89,7 @@ def test_validate_feed_url_sends_user_agent( with patch( "custom_components.feedparser.config_flow.fetch_feed", - return_value=feed.read_bytes(), + return_value=FetchedFeed(feed.read_bytes(), feed.as_uri()), ) as fetch: FeedparserConfigFlow._validate_feed_url(*args) # noqa: SLF001 fetch.assert_called_once_with(feed.as_uri(), expected, "") diff --git a/tests/test_hardening.py b/tests/test_hardening.py index b6600e48..214c479f 100644 --- a/tests/test_hardening.py +++ b/tests/test_hardening.py @@ -2,7 +2,10 @@ from __future__ import annotations +import threading import warnings +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from unittest.mock import patch @@ -78,6 +81,65 @@ def test_non_utf8_feed_is_decoded_from_xml_prolog(tmp_path: Path) -> None: assert sensor.feed_entries[0]["title"] == "Café crème" +RELATIVE_FEED = _rss( + "a/news/1" + '<img src="/img/1.jpg"> hi' + "bitem/2", +).encode() + + +class _FeedHandler(BaseHTTPRequestHandler): + """Serve RELATIVE_FEED at /feed/rss.xml, redirected to from /old.xml.""" + + def do_GET(self) -> None: + if self.path == "/old.xml": + self.send_response(301) + self.send_header("Location", "/feed/rss.xml") + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "application/rss+xml") + self.end_headers() + self.wfile.write(RELATIVE_FEED) + + def log_message(self, *_: object) -> None: + """Keep the test output quiet.""" + + +@pytest.fixture +def feed_server(socket_enabled: None) -> Iterator[str]: # noqa: ARG001 + """Serve RELATIVE_FEED over HTTP and yield the server's base URL.""" + server = ThreadingHTTPServer(("127.0.0.1", 0), _FeedHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +def test_relative_urls_are_made_absolute(feed_server: str) -> None: + """Relative links and images resolve against the feed's final URL (#183).""" + sensor = ReaderHarness( + feed=f"{feed_server}/old.xml", + name="test", + date_format=DATE_FORMAT, + local_time=False, + show_topn=10, + remove_summary_image=False, + inclusions=["title", "link", "image", "summary"], + exclusions=[], + ) + sensor.update() + first, second = sensor.feed_entries + assert first["link"] == f"{feed_server}/news/1" + assert first["image"] == f"{feed_server}/img/1.jpg" + assert first["summary"] == f' hi' + # resolved against where the redirect led, not the configured URL + assert second["link"] == f"{feed_server}/feed/item/2" + + # --- dates -------------------------------------------------------------------- diff --git a/tests/test_proxy.py b/tests/test_proxy.py index dda017b8..76d6fc52 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -236,15 +236,15 @@ def test_fetch_passes_proxy_for_both_schemes() -> None: def test_fetch_through_http_proxy(http_proxy: _Server) -> None: """A feed this machine cannot resolve is fetched through an HTTP proxy.""" - content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy)) - assert content == FEED + fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy)) + assert fetched.content == FEED assert http_proxy.seen == [f"GET {BLOCKED_URL} HTTP/1.1"] def test_fetch_through_socks5h_proxy(socks_proxy: _Server) -> None: """socks5h hands the host name to the proxy, so blocked DNS does not matter.""" - content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy)) - assert content == FEED + fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy)) + assert fetched.content == FEED assert socks_proxy.seen == ["1 blocked.invalid:80"] @@ -253,8 +253,8 @@ def test_file_feed_ignores_proxy(tmp_path: Path) -> None: feed = tmp_path / "feed.xml" feed.write_bytes(FEED) # an unusable proxy: the fetch would fail if it were used - content = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9") - assert content == FEED + fetched = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9") + assert fetched.content == FEED # --- Home Assistant flows -----------------------------------------------------