diff --git a/custom_components/feedparser/config_flow.py b/custom_components/feedparser/config_flow.py index 9088b97f..b6f33349 100644 --- a/custom_components/feedparser/config_flow.py +++ b/custom_components/feedparser/config_flow.py @@ -7,7 +7,6 @@ from datetime import timedelta from typing import Any, cast -import feedparser # type: ignore[import] import requests from homeassistant.config_entries import ( SOURCE_IMPORT, @@ -46,7 +45,7 @@ ENTRY_VERSION, ) from .fetch import USER_AGENT, fetch_feed, is_valid_proxy -from .reader import compile_filter +from .reader import compile_filter, parse_feed from .source import ( URL_ENTITY_DOMAINS, InvalidFeedUrlError, @@ -626,7 +625,7 @@ def _validate_feed_url( Uses the same request (User-Agent, proxy, timeouts) as the sensor so a feed that passes here also works at runtime. """ - parsed = feedparser.parse(fetch_feed(feed_url, user_agent, proxy)) + parsed = parse_feed(fetch_feed(feed_url, user_agent, proxy)) # A real feed exposes a version (rss20, atom10, ...) even when it currently # has no entries; an HTML page or random document does not. if not parsed.get("version") and not parsed.get("entries"): diff --git a/custom_components/feedparser/fetch.py b/custom_components/feedparser/fetch.py index b5042c8a..2097398d 100644 --- a/custom_components/feedparser/fetch.py +++ b/custom_components/feedparser/fetch.py @@ -2,6 +2,7 @@ from __future__ import annotations +from typing import NamedTuple from urllib.parse import urlsplit import requests @@ -20,6 +21,14 @@ PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h") +class FetchedFeed(NamedTuple): + """A downloaded feed and the URL it was finally served from.""" + + content: bytes + # after redirects; feedparser resolves the feed's relative links against it + url: str + + def is_valid_proxy(proxy: str) -> bool: """Return True if `proxy` looks like scheme://[user:pass@]host:port.""" try: @@ -57,8 +66,12 @@ def redact_text(text: str, proxy: str) -> str: return text.replace(proxy, redact_proxy(proxy)) -def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes: - """Download a feed and return its raw bytes. +def fetch_feed( + url: str, + user_agent: str = USER_AGENT, + proxy: str = "", +) -> FetchedFeed: + """Download a feed and return its raw bytes and final URL. Raw bytes (not ``Response.text``) are returned so that feedparser can detect the document encoding from the XML prolog instead of relying on requests' guess. @@ -74,4 +87,4 @@ def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes proxies = {"http": proxy, "https": proxy} if proxy else None response = session.get(url, timeout=REQUEST_TIMEOUT, proxies=proxies) response.raise_for_status() - return response.content + return FetchedFeed(response.content, response.url or url) diff --git a/custom_components/feedparser/reader.py b/custom_components/feedparser/reader.py index ebb0a252..d33750b7 100644 --- a/custom_components/feedparser/reader.py +++ b/custom_components/feedparser/reader.py @@ -15,13 +15,13 @@ import warnings from collections.abc import Mapping from datetime import UTC, datetime, timezone, tzinfo -from typing import Protocol, cast +from typing import Any, Protocol, cast import feedparser # type: ignore[import] from dateutil import parser, tz from homeassistant.util import dt -from .fetch import USER_AGENT, fetch_feed, redact_proxy +from .fetch import USER_AGENT, FetchedFeed, fetch_feed, redact_proxy _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -52,6 +52,19 @@ def _without_offset_name(value: datetime) -> tzinfo | None: return tz.tzoffset(None, offset) +def parse_feed(fetched: FetchedFeed) -> Any: # noqa: ANN401 - feedparser is untyped + """Parse a downloaded feed, resolving relative URLs against where it came from. + + feedparser only knows the document's location when it downloads the feed + itself. Given bytes, it needs the URL as ``Content-Location`` to make + relative links, images and ```` in summaries absolute. + """ + return feedparser.parse( + fetched.content, + response_headers={"content-location": fetched.url}, + ) + + # Entry fields searched by the `match` / `exclude_match` filters. RSS # is exposed by feedparser as `summary`. FILTER_KEYS = ("title", "summary") @@ -176,7 +189,7 @@ def read(self: FeedReader, url: str | None = None) -> list[dict[str, object]]: _LOGGER.debug("Feed %s: Polling feed data from %s", self.name, url) parsed_feed = cast( "ParsedFeed", - feedparser.parse(fetch_feed(url, self._user_agent, self.proxy)), + parse_feed(fetch_feed(url, self._user_agent, self.proxy)), ) if not parsed_feed.entries: _LOGGER.warning("Feed %s: No data received.", self.name) diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py index e8468661..6064d627 100644 --- a/tests/test_config_flow.py +++ b/tests/test_config_flow.py @@ -16,7 +16,7 @@ _user_agent_from_input, ) from custom_components.feedparser.const import CONF_SCAN_INTERVAL, CONF_USER_AGENT -from custom_components.feedparser.fetch import USER_AGENT +from custom_components.feedparser.fetch import USER_AGENT, FetchedFeed def test_options_flow_reloads_config_entry() -> None: @@ -89,7 +89,7 @@ def test_validate_feed_url_sends_user_agent( with patch( "custom_components.feedparser.config_flow.fetch_feed", - return_value=feed.read_bytes(), + return_value=FetchedFeed(feed.read_bytes(), feed.as_uri()), ) as fetch: FeedparserConfigFlow._validate_feed_url(*args) # noqa: SLF001 fetch.assert_called_once_with(feed.as_uri(), expected, "") diff --git a/tests/test_hardening.py b/tests/test_hardening.py index 432904dc..2d2a67ce 100644 --- a/tests/test_hardening.py +++ b/tests/test_hardening.py @@ -2,8 +2,11 @@ from __future__ import annotations +import threading import warnings import zoneinfo +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any from unittest.mock import patch @@ -83,6 +86,65 @@ def test_non_utf8_feed_is_decoded_from_xml_prolog(tmp_path: Path) -> None: assert sensor.feed_entries[0]["title"] == "Café crème" +RELATIVE_FEED = _rss( + "a/news/1" + '<img src="/img/1.jpg"> hi' + "bitem/2", +).encode() + + +class _FeedHandler(BaseHTTPRequestHandler): + """Serve RELATIVE_FEED at /feed/rss.xml, redirected to from /old.xml.""" + + def do_GET(self) -> None: + if self.path == "/old.xml": + self.send_response(301) + self.send_header("Location", "/feed/rss.xml") + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "application/rss+xml") + self.end_headers() + self.wfile.write(RELATIVE_FEED) + + def log_message(self, *_: object) -> None: + """Keep the test output quiet.""" + + +@pytest.fixture +def feed_server(socket_enabled: None) -> Iterator[str]: # noqa: ARG001 + """Serve RELATIVE_FEED over HTTP and yield the server's base URL.""" + server = ThreadingHTTPServer(("127.0.0.1", 0), _FeedHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +def test_relative_urls_are_made_absolute(feed_server: str) -> None: + """Relative links and images resolve against the feed's final URL (#183).""" + sensor = ReaderHarness( + feed=f"{feed_server}/old.xml", + name="test", + date_format=DATE_FORMAT, + local_time=False, + show_topn=10, + remove_summary_image=False, + inclusions=["title", "link", "image", "summary"], + exclusions=[], + ) + sensor.update() + first, second = sensor.feed_entries + assert first["link"] == f"{feed_server}/news/1" + assert first["image"] == f"{feed_server}/img/1.jpg" + assert first["summary"] == f' hi' + # resolved against where the redirect led, not the configured URL + assert second["link"] == f"{feed_server}/feed/item/2" + + # --- dates -------------------------------------------------------------------- diff --git a/tests/test_proxy.py b/tests/test_proxy.py index dda017b8..76d6fc52 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -236,15 +236,15 @@ def test_fetch_passes_proxy_for_both_schemes() -> None: def test_fetch_through_http_proxy(http_proxy: _Server) -> None: """A feed this machine cannot resolve is fetched through an HTTP proxy.""" - content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy)) - assert content == FEED + fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy)) + assert fetched.content == FEED assert http_proxy.seen == [f"GET {BLOCKED_URL} HTTP/1.1"] def test_fetch_through_socks5h_proxy(socks_proxy: _Server) -> None: """socks5h hands the host name to the proxy, so blocked DNS does not matter.""" - content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy)) - assert content == FEED + fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy)) + assert fetched.content == FEED assert socks_proxy.seen == ["1 blocked.invalid:80"] @@ -253,8 +253,8 @@ def test_file_feed_ignores_proxy(tmp_path: Path) -> None: feed = tmp_path / "feed.xml" feed.write_bytes(FEED) # an unusable proxy: the fetch would fail if it were used - content = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9") - assert content == FEED + fetched = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9") + assert fetched.content == FEED # --- Home Assistant flows -----------------------------------------------------