diff --git a/custom_components/feedparser/config_flow.py b/custom_components/feedparser/config_flow.py
index 9088b97f..b6f33349 100644
--- a/custom_components/feedparser/config_flow.py
+++ b/custom_components/feedparser/config_flow.py
@@ -7,7 +7,6 @@
from datetime import timedelta
from typing import Any, cast
-import feedparser # type: ignore[import]
import requests
from homeassistant.config_entries import (
SOURCE_IMPORT,
@@ -46,7 +45,7 @@
ENTRY_VERSION,
)
from .fetch import USER_AGENT, fetch_feed, is_valid_proxy
-from .reader import compile_filter
+from .reader import compile_filter, parse_feed
from .source import (
URL_ENTITY_DOMAINS,
InvalidFeedUrlError,
@@ -626,7 +625,7 @@ def _validate_feed_url(
Uses the same request (User-Agent, proxy, timeouts) as the sensor so a feed
that passes here also works at runtime.
"""
- parsed = feedparser.parse(fetch_feed(feed_url, user_agent, proxy))
+ parsed = parse_feed(fetch_feed(feed_url, user_agent, proxy))
# A real feed exposes a version (rss20, atom10, ...) even when it currently
# has no entries; an HTML page or random document does not.
if not parsed.get("version") and not parsed.get("entries"):
diff --git a/custom_components/feedparser/fetch.py b/custom_components/feedparser/fetch.py
index b5042c8a..2097398d 100644
--- a/custom_components/feedparser/fetch.py
+++ b/custom_components/feedparser/fetch.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from typing import NamedTuple
from urllib.parse import urlsplit
import requests
@@ -20,6 +21,14 @@
PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")
+class FetchedFeed(NamedTuple):
+ """A downloaded feed and the URL it was finally served from."""
+
+ content: bytes
+ # after redirects; feedparser resolves the feed's relative links against it
+ url: str
+
+
def is_valid_proxy(proxy: str) -> bool:
"""Return True if `proxy` looks like scheme://[user:pass@]host:port."""
try:
@@ -57,8 +66,12 @@ def redact_text(text: str, proxy: str) -> str:
return text.replace(proxy, redact_proxy(proxy))
-def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes:
- """Download a feed and return its raw bytes.
+def fetch_feed(
+ url: str,
+ user_agent: str = USER_AGENT,
+ proxy: str = "",
+) -> FetchedFeed:
+ """Download a feed and return its raw bytes and final URL.
Raw bytes (not ``Response.text``) are returned so that feedparser can detect the
document encoding from the XML prolog instead of relying on requests' guess.
@@ -74,4 +87,4 @@ def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes
proxies = {"http": proxy, "https": proxy} if proxy else None
response = session.get(url, timeout=REQUEST_TIMEOUT, proxies=proxies)
response.raise_for_status()
- return response.content
+ return FetchedFeed(response.content, response.url or url)
diff --git a/custom_components/feedparser/reader.py b/custom_components/feedparser/reader.py
index ebb0a252..d33750b7 100644
--- a/custom_components/feedparser/reader.py
+++ b/custom_components/feedparser/reader.py
@@ -15,13 +15,13 @@
import warnings
from collections.abc import Mapping
from datetime import UTC, datetime, timezone, tzinfo
-from typing import Protocol, cast
+from typing import Any, Protocol, cast
import feedparser # type: ignore[import]
from dateutil import parser, tz
from homeassistant.util import dt
-from .fetch import USER_AGENT, fetch_feed, redact_proxy
+from .fetch import USER_AGENT, FetchedFeed, fetch_feed, redact_proxy
_LOGGER: logging.Logger = logging.getLogger(__name__)
@@ -52,6 +52,19 @@ def _without_offset_name(value: datetime) -> tzinfo | None:
return tz.tzoffset(None, offset)
+def parse_feed(fetched: FetchedFeed) -> Any: # noqa: ANN401 - feedparser is untyped
+ """Parse a downloaded feed, resolving relative URLs against where it came from.
+
+ feedparser only knows the document's location when it downloads the feed
+ itself. Given bytes, it needs the URL as ``Content-Location`` to make
+ relative links, images and ``
`` in summaries absolute.
+ """
+ return feedparser.parse(
+ fetched.content,
+ response_headers={"content-location": fetched.url},
+ )
+
+
# Entry fields searched by the `match` / `exclude_match` filters. RSS
# is exposed by feedparser as `summary`.
FILTER_KEYS = ("title", "summary")
@@ -176,7 +189,7 @@ def read(self: FeedReader, url: str | None = None) -> list[dict[str, object]]:
_LOGGER.debug("Feed %s: Polling feed data from %s", self.name, url)
parsed_feed = cast(
"ParsedFeed",
- feedparser.parse(fetch_feed(url, self._user_agent, self.proxy)),
+ parse_feed(fetch_feed(url, self._user_agent, self.proxy)),
)
if not parsed_feed.entries:
_LOGGER.warning("Feed %s: No data received.", self.name)
diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py
index e8468661..6064d627 100644
--- a/tests/test_config_flow.py
+++ b/tests/test_config_flow.py
@@ -16,7 +16,7 @@
_user_agent_from_input,
)
from custom_components.feedparser.const import CONF_SCAN_INTERVAL, CONF_USER_AGENT
-from custom_components.feedparser.fetch import USER_AGENT
+from custom_components.feedparser.fetch import USER_AGENT, FetchedFeed
def test_options_flow_reloads_config_entry() -> None:
@@ -89,7 +89,7 @@ def test_validate_feed_url_sends_user_agent(
with patch(
"custom_components.feedparser.config_flow.fetch_feed",
- return_value=feed.read_bytes(),
+ return_value=FetchedFeed(feed.read_bytes(), feed.as_uri()),
) as fetch:
FeedparserConfigFlow._validate_feed_url(*args) # noqa: SLF001
fetch.assert_called_once_with(feed.as_uri(), expected, "")
diff --git a/tests/test_hardening.py b/tests/test_hardening.py
index 432904dc..2d2a67ce 100644
--- a/tests/test_hardening.py
+++ b/tests/test_hardening.py
@@ -2,8 +2,11 @@
from __future__ import annotations
+import threading
import warnings
import zoneinfo
+from collections.abc import Iterator
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
from unittest.mock import patch
@@ -83,6 +86,65 @@ def test_non_utf8_feed_is_decoded_from_xml_prolog(tmp_path: Path) -> None:
assert sensor.feed_entries[0]["title"] == "Café crème"
+RELATIVE_FEED = _rss(
+ "- a/news/1"
+ '<img src="/img/1.jpg"> hi
'
+ "- bitem/2
",
+).encode()
+
+
+class _FeedHandler(BaseHTTPRequestHandler):
+ """Serve RELATIVE_FEED at /feed/rss.xml, redirected to from /old.xml."""
+
+ def do_GET(self) -> None:
+ if self.path == "/old.xml":
+ self.send_response(301)
+ self.send_header("Location", "/feed/rss.xml")
+ self.end_headers()
+ return
+ self.send_response(200)
+ self.send_header("Content-Type", "application/rss+xml")
+ self.end_headers()
+ self.wfile.write(RELATIVE_FEED)
+
+ def log_message(self, *_: object) -> None:
+ """Keep the test output quiet."""
+
+
+@pytest.fixture
+def feed_server(socket_enabled: None) -> Iterator[str]: # noqa: ARG001
+ """Serve RELATIVE_FEED over HTTP and yield the server's base URL."""
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _FeedHandler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.server_address[1]}"
+ finally:
+ server.shutdown()
+ server.server_close()
+
+
+def test_relative_urls_are_made_absolute(feed_server: str) -> None:
+ """Relative links and images resolve against the feed's final URL (#183)."""
+ sensor = ReaderHarness(
+ feed=f"{feed_server}/old.xml",
+ name="test",
+ date_format=DATE_FORMAT,
+ local_time=False,
+ show_topn=10,
+ remove_summary_image=False,
+ inclusions=["title", "link", "image", "summary"],
+ exclusions=[],
+ )
+ sensor.update()
+ first, second = sensor.feed_entries
+ assert first["link"] == f"{feed_server}/news/1"
+ assert first["image"] == f"{feed_server}/img/1.jpg"
+ assert first["summary"] == f'
hi'
+ # resolved against where the redirect led, not the configured URL
+ assert second["link"] == f"{feed_server}/feed/item/2"
+
+
# --- dates --------------------------------------------------------------------
diff --git a/tests/test_proxy.py b/tests/test_proxy.py
index dda017b8..76d6fc52 100644
--- a/tests/test_proxy.py
+++ b/tests/test_proxy.py
@@ -236,15 +236,15 @@ def test_fetch_passes_proxy_for_both_schemes() -> None:
def test_fetch_through_http_proxy(http_proxy: _Server) -> None:
"""A feed this machine cannot resolve is fetched through an HTTP proxy."""
- content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy))
- assert content == FEED
+ fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy))
+ assert fetched.content == FEED
assert http_proxy.seen == [f"GET {BLOCKED_URL} HTTP/1.1"]
def test_fetch_through_socks5h_proxy(socks_proxy: _Server) -> None:
"""socks5h hands the host name to the proxy, so blocked DNS does not matter."""
- content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy))
- assert content == FEED
+ fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy))
+ assert fetched.content == FEED
assert socks_proxy.seen == ["1 blocked.invalid:80"]
@@ -253,8 +253,8 @@ def test_file_feed_ignores_proxy(tmp_path: Path) -> None:
feed = tmp_path / "feed.xml"
feed.write_bytes(FEED)
# an unusable proxy: the fetch would fail if it were used
- content = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9")
- assert content == FEED
+ fetched = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9")
+ assert fetched.content == FEED
# --- Home Assistant flows -----------------------------------------------------