perf(fetch): cache ESPN scoreboard windows without the parts nothing reads

src/background_data_service.py

@@ -34,6 +34,7 @@
     plugin_scope,
     share_connection_pool,
 )
+from src.common.espn_payload import is_espn_scoreboard_url, slim_scoreboard_payload
 from src.common.espn_dates import (
     RANGE_RETRY_SECONDS,
     _note_range_rejected,
@@ -83,6 +84,10 @@ class FetchRequest:
     # the cache with the callbacks suppressed -- joiners waiting forever for a
     # fetch that did, in fact, succeed.
     commit_claimed: bool = False
+    # Trim an ESPN scoreboard response before it is cached and delivered
+    # (src/common/espn_payload.py). Set by whoever created the request; a
+    # submitter that joins the fetch gets the same payload.
+    slim_payload: bool = True
     result: Optional[Any] = None
     error: Optional[str] = None
     # The plugin that submitted the request, so the fetch service counts the
@@ -249,7 +254,8 @@ def submit_fetch_request(self,
                            timeout: Optional[int] = None,
                            max_retries: int = 3,
                            priority: int = 1,
-                           callback: Optional[Callable] = None) -> str:
+                           callback: Optional[Callable] = None,
+                           slim_payload: bool = True) -> str:
         """
         Submit a background fetch request.
 
@@ -265,6 +271,11 @@ def submit_fetch_request(self,
             priority: Accepted for compatibility and ignored; requests run in
                 submission order.
             callback: Optional callback function when request completes
+            slim_payload: Drop the parts of an ESPN scoreboard response no
+                scoreboard reads (stat leaders, athlete cards, links,
+                headlines, highlights) before caching it; see
+                src/common/espn_payload.py. Only ESPN /scoreboard URLs are
+                touched. Pass False to cache the response whole.
 
         Returns:
             Request ID for tracking the fetch operation
@@ -336,6 +347,7 @@ def submit_fetch_request(self,
             priority=priority,
             callback=callback,
             owner=owner,
+            slim_payload=slim_payload,
         )
 
         with self._lock:
@@ -497,6 +509,13 @@ def _fetch_data_worker_scoped(self, request: FetchRequest) -> FetchResult:
                 )
                 return result
 
+            # Most of an ESPN scoreboard response is never drawn, and the
+            # cached copy stays parsed in the memory tier while it is fresh.
+            # Trimmed before the write so the cache, request.result and the
+            # callbacks all see the same payload. See src/common/espn_payload.py.
+            if request.slim_payload and is_espn_scoreboard_url(request.url):
+                slim_scoreboard_payload(data)
+
             # Cache the data
             self.cache_manager.set(request.cache_key, data)
 

src/common/espn_payload.py

@@ -0,0 +1,97 @@
+"""Drop the parts of an ESPN scoreboard payload no scoreboard reads.
+
+The sports scoreboards cache their Recent/Upcoming window (14 days back, 7
+ahead) as the raw ESPN response, and that record stays parsed in the memory
+cache for as long as it is fresh. Most of it is never drawn. Measured on hdpi
+(2026-10-02) the MLB window was 3.35MB of JSON and 13.5MB of Python objects,
+and the five windows together ~40MB, mostly in:
+
+* ``competitors[].leaders`` / ``competitions[].leaders`` -- per-team and
+  per-game stat leaders (28% of the MLB window)
+* ``competitors[].team.links`` / ``event.links`` -- web and app URLs
+* ``status.featuredAthletes`` and ``competitors[].probables`` -- athlete
+  cards with headshots and season stats
+* ``competitions[].headlines`` / ``highlights`` -- article and video blurbs
+  (28% of the college-football window)
+* ``competitions[].geoBroadcasts``
+
+None of those keys is read by core or by any plugin in ledmatrix-plugins
+(checked 2026-10-02 across every scoreboard, the odds ticker and the
+leaderboard), while everything that is read -- odds, records, linescores,
+situation, statistics, notes, broadcasts, venue -- is kept. Dropping them
+takes the five windows from ~40MB to ~12MB of parsed objects and the files from
+10.6MB to 3.0MB, so the reads that parse an expired window on the render
+thread get 3-4x cheaper too.
+
+:func:`slim_scoreboard_payload` changes the payload in place, and only ever
+removes the keys listed here: anything it does not know about is left alone.
+"""
+
+from typing import Any, Dict
+from urllib.parse import urlsplit
+
+# Per level of the payload, the keys removed. Kept deliberately explicit:
+# adding a key here means checking that nothing reads it first.
+_EVENT_DROP = ("links",)
+_COMPETITION_DROP = ("leaders", "headlines", "highlights", "geoBroadcasts")
+_STATUS_DROP = ("featuredAthletes",)
+_COMPETITOR_DROP = ("leaders", "probables")
+_TEAM_DROP = ("links",)
+
+
+def is_espn_scoreboard_url(url: Any) -> bool:
+    """Whether ``url`` is an ESPN site-API scoreboard endpoint."""
+    if not isinstance(url, str):
+        return False
+    try:
+        parts = urlsplit(url)
+    except ValueError:
+        return False
+    host = (parts.hostname or "").lower()
+    if host != "espn.com" and not host.endswith(".espn.com"):
+        return False
+    return parts.path.rstrip("/").endswith("/scoreboard")
+
+
+def _drop(obj: Any, keys) -> None:
+    if isinstance(obj, dict):
+        for key in keys:
+            obj.pop(key, None)
+
+
+def slim_scoreboard_payload(payload: Any) -> Any:
+    """Remove the unread parts of an ESPN scoreboard payload, in place.
+
+    Returns ``payload`` for convenience. Anything that is not shaped like a
+    scoreboard (not a dict, no ``events`` list, odd entries) is passed over
+    untouched rather than raising.
+    """
+    if not isinstance(payload, dict):
+        return payload
+    events = payload.get("events")
+    if not isinstance(events, list):
+        return payload
+    for event in events:
+        if not isinstance(event, dict):
+            continue
+        _drop(event, _EVENT_DROP)
+        competitions = event.get("competitions")
+        if not isinstance(competitions, list):
+            continue
+        for competition in competitions:
+            if not isinstance(competition, dict):
+                continue
+            _drop(competition, _COMPETITION_DROP)
+            _drop(competition.get("status"), _STATUS_DROP)
+            competitors = competition.get("competitors")
+            if not isinstance(competitors, list):
+                continue
+            for competitor in competitors:
+                if not isinstance(competitor, dict):
+                    continue
+                _drop(competitor, _COMPETITOR_DROP)
+                _drop(competitor.get("team"), _TEAM_DROP)
+    return payload
+
+
+__all__ = ["is_espn_scoreboard_url", "slim_scoreboard_payload"]

test/test_espn_payload.py

@@ -0,0 +1,162 @@
+"""Tests for src/common/espn_payload.py and its use by BackgroundDataService."""
+
+import copy
+import time
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+
+from src.background_data_service import BackgroundDataService, shutdown_background_service
+from src.common.espn_payload import is_espn_scoreboard_url, slim_scoreboard_payload
+
+SCOREBOARD = "https://site.api.espn.com/apis/site/v2/sports/baseball/mlb/scoreboard"
+
+
+def _event():
+    """One event carrying every key the slimming drops and a sample of the
+    keys scoreboards read, at the depth ESPN puts them."""
+    competitor = {
+        "id": "10",
+        "homeAway": "home",
+        "score": "5",
+        "team": {"abbreviation": "NYY", "logo": "https://a/l.png",
+                 "links": [{"href": "https://espn.com/team"}]},
+        "records": [{"summary": "90-60"}],
+        "linescores": [{"value": 1}],
+        "statistics": [{"name": "hits", "displayValue": "9"}],
+        "leaders": [{"name": "avg", "leaders": [{"athlete": {"id": "1"}}]}],
+        "probables": [{"athlete": {"id": "2"}, "statistics": []}],
+    }
+    return {
+        "id": "401",
+        "date": "2026-10-01T23:05Z",
+        "links": [{"href": "https://espn.com/game"}],
+        "status": {"type": {"state": "post"}},
+        "competitions": [{
+            "status": {"type": {"state": "post", "shortDetail": "Final"},
+                       "featuredAthletes": [{"athlete": {"id": "3"}}]},
+            "competitors": [competitor, dict(copy.deepcopy(competitor), homeAway="away")],
+            "odds": [{"details": "NYY -150", "overUnder": 8.5}],
+            "situation": {"outs": 2},
+            "notes": [{"headline": "Game 1"}],
+            "broadcasts": [{"names": ["FOX"]}],
+            "venue": {"fullName": "Yankee Stadium"},
+            "leaders": [{"name": "hits"}],
+            "headlines": [{"description": "recap"}],
+            "highlights": [{"links": {"source": {}}}],
+            "geoBroadcasts": [{"media": {"shortName": "FOX"}}],
+        }],
+    }
+
+
+class TestSlimScoreboardPayload:
+    def test_drops_exactly_the_listed_keys(self):
+        payload = {"leagues": [{"id": "10"}], "events": [_event()]}
+        slim_scoreboard_payload(payload)
+        event = payload["events"][0]
+        competition = event["competitions"][0]
+        assert "links" not in event
+        for key in ("leaders", "headlines", "highlights", "geoBroadcasts"):
+            assert key not in competition
+        assert "featuredAthletes" not in competition["status"]
+        for competitor in competition["competitors"]:
+            assert "leaders" not in competitor
+            assert "probables" not in competitor
+            assert "links" not in competitor["team"]
+
+    def test_keeps_everything_else_unchanged(self):
+        """Removing the dropped keys from the original by hand gives exactly
+        the slimmed payload: nothing else moved, changed or went missing."""
+        original = {"leagues": [{"id": "10"}], "events": [_event(), _event()]}
+        expected = copy.deepcopy(original)
+        for event in expected["events"]:
+            del event["links"]
+            competition = event["competitions"][0]
+            for key in ("leaders", "headlines", "highlights", "geoBroadcasts"):
+                del competition[key]
+            del competition["status"]["featuredAthletes"]
+            for competitor in competition["competitors"]:
+                del competitor["leaders"], competitor["probables"]
+                del competitor["team"]["links"]
+        assert slim_scoreboard_payload(original) == expected
+
+    def test_in_place_and_returns_payload(self):
+        payload = {"events": [_event()]}
+        assert slim_scoreboard_payload(payload) is payload
+
+    @pytest.mark.parametrize("payload", [
+        None, [], "x", {}, {"events": None}, {"events": "x"},
+        {"events": [None, 1, "x", {"competitions": None}]},
+        {"events": [{"competitions": [None, {"status": None, "competitors": None}]}]},
+        {"events": [{"competitions": [{"competitors": [None, {"team": None}]}]}]},
+    ])
+    def test_odd_shapes_pass_through(self, payload):
+        before = copy.deepcopy(payload)
+        assert slim_scoreboard_payload(payload) == before
+
+
+class TestIsEspnScoreboardUrl:
+    @pytest.mark.parametrize("url", [
+        SCOREBOARD,
+        SCOREBOARD + "/",
+        "http://site.api.espn.com/apis/site/v2/sports/football/college-football/scoreboard",
+    ])
+    def test_scoreboards(self, url):
+        assert is_espn_scoreboard_url(url)
+
+    @pytest.mark.parametrize("url", [
+        None, "", 12,
+        "https://site.api.espn.com/apis/site/v2/sports/baseball/mlb/teams",
+        "https://site.api.espn.com/apis/site/v2/sports/football/nfl/summary",
+        "https://example.com/scoreboard",
+        "https://espn.com.evil.example/apis/x/scoreboard",
+        "https://notespn.com/apis/x/scoreboard",
+    ])
+    def test_not_scoreboards(self, url):
+        assert not is_espn_scoreboard_url(url)
+
+
+@pytest.fixture
+def service():
+    shutdown_background_service()
+    cache = MagicMock()
+    cache.get.return_value = None
+    svc = BackgroundDataService(cache, max_workers=1, request_timeout=5)
+    yield svc
+    svc.shutdown(wait=False)
+    shutdown_background_service()
+
+
+def _run(service, url, **kwargs):
+    response = Mock(status_code=200)
+    response.json.return_value = {"events": [_event()]}
+    response.raise_for_status.return_value = None
+    delivered = []
+    with patch.object(service.session, "get", return_value=response):
+        req_id = service.submit_fetch_request(
+            sport="mlb", year=2026, url=url, cache_key="mlb_schedule_window_14_7",
+            callback=lambda result: delivered.append(result.data), **kwargs)
+        deadline = time.time() + 5
+        while not service.is_request_complete(req_id) and time.time() < deadline:
+            time.sleep(0.02)
+    cached = service.cache_manager.set.call_args[0][1]
+    return cached, delivered
+
+
+class TestBackgroundServiceSlims:
+    def test_espn_scoreboard_is_cached_and_delivered_slimmed(self, service):
+        cached, delivered = _run(service, SCOREBOARD)
+        competition = cached["events"][0]["competitions"][0]
+        assert "leaders" not in competition
+        assert "probables" not in competition["competitors"][0]
+        assert competition["odds"] and competition["situation"]
+        # The callback sees the very payload that was cached.
+        assert delivered and delivered[0] is cached
+
+    def test_opt_out_caches_whole_response(self, service):
+        cached, _ = _run(service, SCOREBOARD, slim_payload=False)
+        assert cached == {"events": [_event()]}
+
+    def test_other_urls_untouched(self, service):
+        cached, _ = _run(service, "https://example.com/feed")
+        assert cached == {"events": [_event()]}