Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions custom_components/feedparser/config_flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
from datetime import timedelta
from typing import Any, cast

import feedparser # type: ignore[import]
import requests
from homeassistant.config_entries import (
SOURCE_IMPORT,
Expand Down Expand Up @@ -46,7 +45,7 @@
ENTRY_VERSION,
)
from .fetch import USER_AGENT, fetch_feed, is_valid_proxy
from .reader import compile_filter
from .reader import compile_filter, parse_feed
from .source import (
URL_ENTITY_DOMAINS,
InvalidFeedUrlError,
Expand Down Expand Up @@ -626,7 +625,7 @@ def _validate_feed_url(
Uses the same request (User-Agent, proxy, timeouts) as the sensor so a feed
that passes here also works at runtime.
"""
parsed = feedparser.parse(fetch_feed(feed_url, user_agent, proxy))
parsed = parse_feed(fetch_feed(feed_url, user_agent, proxy))
# A real feed exposes a version (rss20, atom10, ...) even when it currently
# has no entries; an HTML page or random document does not.
if not parsed.get("version") and not parsed.get("entries"):
Expand Down
19 changes: 16 additions & 3 deletions custom_components/feedparser/fetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

from typing import NamedTuple
from urllib.parse import urlsplit

import requests
Expand All @@ -20,6 +21,14 @@
PROXY_SCHEMES = ("http", "https", "socks4", "socks4a", "socks5", "socks5h")


class FetchedFeed(NamedTuple):
"""A downloaded feed and the URL it was finally served from."""

content: bytes
# after redirects; feedparser resolves the feed's relative links against it
url: str


def is_valid_proxy(proxy: str) -> bool:
"""Return True if `proxy` looks like scheme://[user:pass@]host:port."""
try:
Expand Down Expand Up @@ -57,8 +66,12 @@ def redact_text(text: str, proxy: str) -> str:
return text.replace(proxy, redact_proxy(proxy))


def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes:
"""Download a feed and return its raw bytes.
def fetch_feed(
url: str,
user_agent: str = USER_AGENT,
proxy: str = "",
) -> FetchedFeed:
"""Download a feed and return its raw bytes and final URL.

Raw bytes (not ``Response.text``) are returned so that feedparser can detect the
document encoding from the XML prolog instead of relying on requests' guess.
Expand All @@ -74,4 +87,4 @@ def fetch_feed(url: str, user_agent: str = USER_AGENT, proxy: str = "") -> bytes
proxies = {"http": proxy, "https": proxy} if proxy else None
response = session.get(url, timeout=REQUEST_TIMEOUT, proxies=proxies)
response.raise_for_status()
return response.content
return FetchedFeed(response.content, response.url or url)
19 changes: 16 additions & 3 deletions custom_components/feedparser/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@
import warnings
from collections.abc import Mapping
from datetime import UTC, datetime, timezone, tzinfo
from typing import Protocol, cast
from typing import Any, Protocol, cast

import feedparser # type: ignore[import]
from dateutil import parser, tz
from homeassistant.util import dt

from .fetch import USER_AGENT, fetch_feed, redact_proxy
from .fetch import USER_AGENT, FetchedFeed, fetch_feed, redact_proxy

_LOGGER: logging.Logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -52,6 +52,19 @@ def _without_offset_name(value: datetime) -> tzinfo | None:
return tz.tzoffset(None, offset)


def parse_feed(fetched: FetchedFeed) -> Any: # noqa: ANN401 - feedparser is untyped
"""Parse a downloaded feed, resolving relative URLs against where it came from.

feedparser only knows the document's location when it downloads the feed
itself. Given bytes, it needs the URL as ``Content-Location`` to make
relative links, images and ``<img src>`` in summaries absolute.
"""
return feedparser.parse(
fetched.content,
response_headers={"content-location": fetched.url},
)


# Entry fields searched by the `match` / `exclude_match` filters. RSS
# <description> is exposed by feedparser as `summary`.
FILTER_KEYS = ("title", "summary")
Expand Down Expand Up @@ -176,7 +189,7 @@ def read(self: FeedReader, url: str | None = None) -> list[dict[str, object]]:
_LOGGER.debug("Feed %s: Polling feed data from %s", self.name, url)
parsed_feed = cast(
"ParsedFeed",
feedparser.parse(fetch_feed(url, self._user_agent, self.proxy)),
parse_feed(fetch_feed(url, self._user_agent, self.proxy)),
)
if not parsed_feed.entries:
_LOGGER.warning("Feed %s: No data received.", self.name)
Expand Down
4 changes: 2 additions & 2 deletions tests/test_config_flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
_user_agent_from_input,
)
from custom_components.feedparser.const import CONF_SCAN_INTERVAL, CONF_USER_AGENT
from custom_components.feedparser.fetch import USER_AGENT
from custom_components.feedparser.fetch import USER_AGENT, FetchedFeed


def test_options_flow_reloads_config_entry() -> None:
Expand Down Expand Up @@ -89,7 +89,7 @@ def test_validate_feed_url_sends_user_agent(

with patch(
"custom_components.feedparser.config_flow.fetch_feed",
return_value=feed.read_bytes(),
return_value=FetchedFeed(feed.read_bytes(), feed.as_uri()),
) as fetch:
FeedparserConfigFlow._validate_feed_url(*args) # noqa: SLF001
fetch.assert_called_once_with(feed.as_uri(), expected, "")
62 changes: 62 additions & 0 deletions tests/test_hardening.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@

from __future__ import annotations

import threading
import warnings
import zoneinfo
from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
from unittest.mock import patch
Expand Down Expand Up @@ -83,6 +86,65 @@ def test_non_utf8_feed_is_decoded_from_xml_prolog(tmp_path: Path) -> None:
assert sensor.feed_entries[0]["title"] == "Café crème"


RELATIVE_FEED = _rss(
"<item><title>a</title><link>/news/1</link>"
'<description>&lt;img src="/img/1.jpg"&gt; hi</description></item>'
"<item><title>b</title><link>item/2</link></item>",
).encode()


class _FeedHandler(BaseHTTPRequestHandler):
"""Serve RELATIVE_FEED at /feed/rss.xml, redirected to from /old.xml."""

def do_GET(self) -> None:
if self.path == "/old.xml":
self.send_response(301)
self.send_header("Location", "/feed/rss.xml")
self.end_headers()
return
self.send_response(200)
self.send_header("Content-Type", "application/rss+xml")
self.end_headers()
self.wfile.write(RELATIVE_FEED)

def log_message(self, *_: object) -> None:
"""Keep the test output quiet."""


@pytest.fixture
def feed_server(socket_enabled: None) -> Iterator[str]: # noqa: ARG001
"""Serve RELATIVE_FEED over HTTP and yield the server's base URL."""
server = ThreadingHTTPServer(("127.0.0.1", 0), _FeedHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield f"http://127.0.0.1:{server.server_address[1]}"
finally:
server.shutdown()
server.server_close()


def test_relative_urls_are_made_absolute(feed_server: str) -> None:
"""Relative links and images resolve against the feed's final URL (#183)."""
sensor = ReaderHarness(
feed=f"{feed_server}/old.xml",
name="test",
date_format=DATE_FORMAT,
local_time=False,
show_topn=10,
remove_summary_image=False,
inclusions=["title", "link", "image", "summary"],
exclusions=[],
)
sensor.update()
first, second = sensor.feed_entries
assert first["link"] == f"{feed_server}/news/1"
assert first["image"] == f"{feed_server}/img/1.jpg"
assert first["summary"] == f'<img src="{feed_server}/img/1.jpg" /> hi'
# resolved against where the redirect led, not the configured URL
assert second["link"] == f"{feed_server}/feed/item/2"


# --- dates --------------------------------------------------------------------


Expand Down
12 changes: 6 additions & 6 deletions tests/test_proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,15 +236,15 @@ def test_fetch_passes_proxy_for_both_schemes() -> None:

def test_fetch_through_http_proxy(http_proxy: _Server) -> None:
"""A feed this machine cannot resolve is fetched through an HTTP proxy."""
content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy))
assert content == FEED
fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("http", http_proxy))
assert fetched.content == FEED
assert http_proxy.seen == [f"GET {BLOCKED_URL} HTTP/1.1"]


def test_fetch_through_socks5h_proxy(socks_proxy: _Server) -> None:
"""socks5h hands the host name to the proxy, so blocked DNS does not matter."""
content = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy))
assert content == FEED
fetched = fetch.fetch_feed(BLOCKED_URL, proxy=_url("socks5h", socks_proxy))
assert fetched.content == FEED
assert socks_proxy.seen == ["1 blocked.invalid:80"]


Expand All @@ -253,8 +253,8 @@ def test_file_feed_ignores_proxy(tmp_path: Path) -> None:
feed = tmp_path / "feed.xml"
feed.write_bytes(FEED)
# an unusable proxy: the fetch would fail if it were used
content = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9")
assert content == FEED
fetched = fetch.fetch_feed(feed.as_uri(), proxy="http://127.0.0.1:9")
assert fetched.content == FEED


# --- Home Assistant flows -----------------------------------------------------
Expand Down
Loading