From 5eecd6f0525769b22910288beb49c1e320a24a77 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:35:46 -0400 Subject: [PATCH] fix(soccer-scoreboard): fetch a league's schedule window once per miss (2.39.3) Every start on ledpi (8 leagues) logged ~90 NameResolutionErrors for site.api.espn.com within a minute, and six hours made 3,812 requests. On a cache miss _fetch_soccer_api_data submitted the window to the core's background service AND fetched the same window on the spot as stand-in data, in both the recent and the upcoming manager; with ESPN rejecting date ranges each window is 29 day requests. A miss now fetches the window once on the calling thread (_fetch_season_directly, which caches it) under a lock per cache key, so the league's other manager waits and reads the cache. Managers share the core's connection pool where the core has one, so connections and DNS lookups are reused across leagues. Live ESPN, 8 leagues, core main: ~440 -> ~260 requests per start, ~470 -> ~250 per hourly refresh; with LEDMatrix #751 39 and ~20. Co-Authored-By: Claude Opus 5.5 --- plugins.json | 7 +- plugins/soccer-scoreboard/CHANGELOG.md | 17 ++ plugins/soccer-scoreboard/manifest.json | 8 +- plugins/soccer-scoreboard/soccer_managers.py | 154 +++++++------- .../test_schedule_fetched_once.py | 198 ++++++++++++++++++ 5 files changed, 299 insertions(+), 85 deletions(-) create mode 100644 plugins/soccer-scoreboard/test_schedule_fetched_once.py diff --git a/plugins.json b/plugins.json index fcd23da6..01b65cb0 100644 --- a/plugins.json +++ b/plugins.json @@ -844,12 +844,11 @@ "plugin_path": "plugins/soccer-scoreboard", "stars": 0, "downloads": 0, - "last_updated": "2026-10-02", + "last_updated": "2026-10-03", "verified": true, "screenshot": "", - "latest_version": "2.39.2", - "ledmatrix_min_version": "3.8.0", - "commit": "886d2038d22a35c962beac0ea261543339cffec9" + "latest_version": "2.39.3", + "ledmatrix_min_version": "3.8.0" }, { "id": "static-image", diff --git a/plugins/soccer-scoreboard/CHANGELOG.md b/plugins/soccer-scoreboard/CHANGELOG.md index 4645dc21..30dae37c 100644 --- a/plugins/soccer-scoreboard/CHANGELOG.md +++ b/plugins/soccer-scoreboard/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [2.39.3] - 2026-10-03 + +### Fixed +- A start or reload no longer floods ESPN. With eight leagues, every start + logged ~90 `NameResolutionError`s for site.api.espn.com within a minute + (plus `update() timed out`), and six hours made 3,812 requests. On a cache + miss `_fetch_soccer_api_data` submitted the league's schedule window to the + core's background service *and* fetched the same window on the spot as + stand-in data (`_get_weeks_data`), in both the recent and the upcoming + manager -- and ESPN's rejection of date ranges makes each window 29 day + requests. A miss now fetches the window once on the calling thread + (`_fetch_season_directly`, which caches it), under a lock per cache key so + the league's other manager waits and reads the cache. +- Every manager shares the core's connection pool (`share_connection_pool`, + on cores that have the fetch service) instead of opening its own, so a + connection -- and its DNS lookup -- is reused across leagues. + ## [2.39.2] - 2026-10-02 ### Fixed diff --git a/plugins/soccer-scoreboard/manifest.json b/plugins/soccer-scoreboard/manifest.json index bab33e9c..07a525ea 100644 --- a/plugins/soccer-scoreboard/manifest.json +++ b/plugins/soccer-scoreboard/manifest.json @@ -1,7 +1,7 @@ { "id": "soccer-scoreboard", "name": "Soccer Scoreboard", - "version": "2.39.2", + "version": "2.39.3", "author": "ChuckBuilds", "description": "Live, recent, and upcoming soccer games across multiple leagues including Premier League, La Liga, Bundesliga, Serie A, Ligue 1, MLS, Liga Portugal, Champions League, Europa League, and FIFA World Cup", "category": "sports", @@ -53,6 +53,12 @@ "soccer_usa.1_upcoming" ], "versions": [ + { + "version": "2.39.3", + "released": "2026-10-03", + "ledmatrix_min_version": "3.8.0", + "notes": "Startup and hourly ESPN traffic cut sharply. A league's schedule window is now fetched once per cache miss: it used to be submitted to the core's background service and fetched again on the spot as stand-in data, by both the recent and the upcoming manager, so each league asked for its window up to four times. A per-window lock makes the league's other manager wait and reuse the cached result, and every manager shares the core's connection pool (on cores that have it), so connections and DNS lookups are reused. Measured against live ESPN with eight leagues on core main: ~440 requests per start became ~260, and ~470 per hourly refresh became ~250; with LEDMatrix core's month-sized date fetches (LEDMatrix #751) a start is 39 requests and a refresh about 20. No display change." + }, { "version": "2.39.2", "released": "2026-10-02", diff --git a/plugins/soccer-scoreboard/soccer_managers.py b/plugins/soccer-scoreboard/soccer_managers.py index bac5dba3..47c11198 100644 --- a/plugins/soccer-scoreboard/soccer_managers.py +++ b/plugins/soccer-scoreboard/soccer_managers.py @@ -7,13 +7,20 @@ import logging import shutil +import threading from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, ClassVar, Dict, Optional import pytz from sports import SportsCore, SportsLive, SportsRecent, SportsUpcoming +try: + # Core's shared connection pool (core #702); older cores have none. + from src.common.fetch_service import share_connection_pool +except ImportError: + share_connection_pool = None + # ESPN API base URL for soccer ESPN_SOCCER_BASE_URL = "https://site.api.espn.com/apis/site/v2/sports/soccer" @@ -42,6 +49,18 @@ class BaseSoccerManager(SportsCore): _shared_data = None _last_shared_update = 0 + # One schedule fetch per cache key at a time, across every manager. A + # league's recent and upcoming managers read the same key and come due in + # the same update(), so without this both missed the cache and both + # fetched the window. + _window_locks: ClassVar[Dict[str, threading.Lock]] = {} + _window_locks_guard: ClassVar[threading.Lock] = threading.Lock() + + @classmethod + def _window_lock(cls, cache_key: str) -> threading.Lock: + with cls._window_locks_guard: + return cls._window_locks.setdefault(cache_key, threading.Lock()) + def __init__(self, config: Dict[str, Any], display_manager, cache_manager, league_key: str): """ Initialize base soccer manager. @@ -68,6 +87,19 @@ def __init__(self, config: Dict[str, Any], display_manager, cache_manager, leagu self.sport = "soccer" self.league = league_key + # Every manager -- three per league -- built its own Session and so + # its own connection pool: each new connection a DNS lookup, all at + # once at startup. On a core with the shared fetch service they now + # share one pool per retry policy (the same policy they mount), so a + # connection one league opened serves the next. + if share_connection_pool is not None: + try: + share_connection_pool( + self.session, self.session.get_adapter("https://").max_retries + ) + except Exception as e: # noqa: BLE001 - its own pool still works + self.logger.debug(f"Could not share the connection pool: {e}") + # National-team flags (FIFA World Cup) live in a dedicated subdirectory so # they never collide with club logos that share an abbreviation — e.g. ESP # (Spain vs Espanyol), POR (Portugal vs Portland Timbers), COL (Colombia vs @@ -131,12 +163,21 @@ def _setup_national_flags_dir(self, club_logo_dir: Path) -> Path: return flags_dir def _fetch_soccer_api_data(self, use_cache: bool = True) -> Optional[Dict]: - """ - Fetches game data for the soccer league using background threading. - Returns cached data immediately if available, otherwise starts background fetch. + """The league's schedule over the configured window: cached, or + fetched once on this thread and cached. + + This used to submit the window to the core's background service *and* + fetch the same window here at once, for "immediate" partial data -- + the same request twice, every time the cache missed. With ESPN + rejecting date ranges, each was 29 day requests per league; eight + leagues at startup made ~450 requests and ~90 NameResolutionErrors + on a Pi. A fetch here costs what the stand-in did and nothing else. + + The window lock makes the league's other manager wait for this + fetch and read its result from the cache instead of repeating it. """ now = datetime.now(pytz.utc) - + # The window the user configured, not a fixed fortnight. This is the # authoritative fetch; leaving it hard-coded meant a widened setting # showed briefly from the stand-in fetch and then vanished when this @@ -144,88 +185,41 @@ def _fetch_soccer_api_data(self, use_cache: bool = True) -> Optional[Dict]: start_date = now - timedelta(days=self.schedule_lookback_days) end_date = now + timedelta(days=self.schedule_lookahead_days) date_str = f"{start_date.strftime('%Y%m%d')}-{end_date.strftime('%Y%m%d')}" - + cache_key = f"soccer_{self.league_key}_schedule_{date_str}" url = f"{ESPN_SOCCER_BASE_URL}/{self.league_key}/scoreboard" - # Check cache first - if use_cache: - cached_data = self.cache_manager.get(cache_key) - if cached_data: - # Validate cached data structure - if isinstance(cached_data, dict) and "events" in cached_data: - self.logger.info(f"Using cached schedule for {self.league_name}") - return cached_data - elif isinstance(cached_data, list): - # Handle old cache format (list of events) - self.logger.info( - f"Using cached schedule for {self.league_name} (legacy format)" - ) - return {"events": cached_data} - else: - self.logger.warning( - f"Invalid cached data format for {self.league_name}: {type(cached_data)}" - ) - # Clear invalid cache - self.cache_manager.delete(cache_key) - - # Start background fetch if service is available - if ( - self.background_service - and self.background_enabled - and self._background_fetches_espn_ranges() - ): - self.logger.info( - f"Starting background fetch for {self.league_name} schedule..." - ) - - def fetch_callback(result): - """Callback when background fetch completes.""" - if result.success: - self.logger.info( - f"Background fetch completed for {self.league_name}: {len(result.data.get('events', []))} events" - ) - else: - self.logger.error( - f"Background fetch failed for {self.league_name}: {result.error}" - ) - - # Get background service configuration - background_config = self.mode_config.get("background_service", {}) - timeout = background_config.get("request_timeout", 30) - max_retries = background_config.get("max_retries", 3) - priority = background_config.get("priority", 2) - - # Submit background fetch request - request_id = self.background_service.submit_fetch_request( - sport="soccer", - year=now.year, - url=url, - cache_key=cache_key, - params={"dates": date_str, "limit": 1000}, - headers=self.headers, - timeout=timeout, - max_retries=max_retries, - priority=priority, - callback=fetch_callback, - ) - - # Track the request - if not hasattr(self, 'background_fetch_requests'): - self.background_fetch_requests = {} - self.background_fetch_requests[date_str] = request_id - - # For immediate response, try to get partial data - partial_data = self._get_weeks_data() - if partial_data: - return partial_data - else: - # No background service, or a core that would send this range to - # ESPN as-is (rejected with 400 since 2026-09-15): fetch it here. + with self._window_lock(cache_key): + if use_cache: + cached = self._cached_window(cache_key) + if cached is not None: + return cached return self._fetch_season_directly( url, date_str, cache_key, f"{self.league_name}" ) + def _cached_window(self, cache_key: str) -> Optional[Dict]: + """The schedule cached under ``cache_key``, or None.""" + cached_data = self.cache_manager.get(cache_key) + if not cached_data: + return None + # Validate cached data structure + if isinstance(cached_data, dict) and "events" in cached_data: + self.logger.info(f"Using cached schedule for {self.league_name}") + return cached_data + if isinstance(cached_data, list): + # Handle old cache format (list of events) + self.logger.info( + f"Using cached schedule for {self.league_name} (legacy format)" + ) + return {"events": cached_data} + self.logger.warning( + f"Invalid cached data format for {self.league_name}: {type(cached_data)}" + ) + # Clear invalid cache + self.cache_manager.delete(cache_key) + return None + def _fetch_data(self) -> Optional[Dict]: """Fetch data using shared data mechanism or direct fetch for live.""" if isinstance(self, SoccerLiveManager): diff --git a/plugins/soccer-scoreboard/test_schedule_fetched_once.py b/plugins/soccer-scoreboard/test_schedule_fetched_once.py new file mode 100644 index 00000000..885f9474 --- /dev/null +++ b/plugins/soccer-scoreboard/test_schedule_fetched_once.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""A league's schedule window is fetched once per cache miss, not four times. + +Found on a Pi running eight leagues (ledpi, 2026-10-03): every start logged +~90 NameResolutionErrors for site.api.espn.com within a minute, and over six +hours soccer made 3,812 requests. ESPN rejects date ranges, so each league's +29-day window was 29 day requests -- and _fetch_soccer_api_data, on a cache +miss, both submitted the window to the core's background service AND fetched +the same window on the spot as "partial data". The recent and upcoming +managers share the cache key but come due in the same update(), so both +missed it and both did both. Measured against live ESPN at startup: ~450 +requests, ~45 in flight at once. + +Now a miss fetches the window once on the calling thread +(_fetch_season_directly, which caches it), under a per-key lock, so the +league's other manager waits and reads the cache. The managers also share the +core's connection pool, so their connections (and DNS lookups) are reused. + +No network: the fetch is replaced by a counting stub. + +Run: /bin/python plugins/soccer-scoreboard/test_schedule_fetched_once.py +""" + +# pylint: disable=protected-access +import logging +import os +import sys +import threading +import time +from pathlib import Path +from unittest.mock import MagicMock + +plugin_dir = Path(__file__).resolve().parent +sys.path.insert(0, str(plugin_dir)) +_core = os.environ.get("LEDMATRIX_CORE") +_candidates = [Path(_core)] if _core else [] +_candidates.append(plugin_dir.parents[2] / "LEDMatrix") +for candidate in _candidates: + if (candidate / "src" / "plugin_system" / "base_plugin.py").exists(): + sys.path.insert(0, str(candidate)) + break +else: + print("SKIP: no LEDMatrix core checkout found (set LEDMATRIX_CORE)") + sys.exit(2) + +import soccer_managers as sm # noqa: E402 + +failures = [] + + +def check(label, ok, detail=None): + print((" PASS " if ok else " FAIL ") + label + + ("" if ok or detail is None else " -- %r" % (detail,))) + if not ok: + failures.append(label) + + +class DictCache: + def __init__(self): + self.data = {} + self.lock = threading.Lock() + + def get(self, key, max_age=300): + with self.lock: + return self.data.get(key) + + def set(self, key, value, ttl=None): + with self.lock: + self.data[key] = value + + def delete(self, key): + with self.lock: + self.data.pop(key, None) + + +def manager(cls, cache, fetches, league="eng.1"): + """A manager with only what the schedule fetch reads; the constructor + pulls in fonts, logos and an ESPN session this test has no use for.""" + obj = cls.__new__(cls) + obj.league_key = league + obj.league_name = sm.LEAGUE_NAMES[league] + obj.logger = logging.getLogger("fetch_once_probe") + obj.cache_manager = cache + obj.schedule_lookback_days = 14 + obj.schedule_lookahead_days = 14 + obj.background_service = MagicMock() + obj.background_enabled = True + obj.mode_config = {} + obj.headers = {} + obj._background_fetches_espn_ranges = lambda: True + obj._get_weeks_data = lambda: fetches.append("partial") or {"events": []} + + def fetch(url, datestring, cache_key, label, ttl=None): + time.sleep(0.05) # long enough for the other manager to arrive + fetches.append(datestring) + data = {"events": [{"id": "1"}]} + cache.set(cache_key, data) + return data + + obj._fetch_season_directly = fetch + return obj + + +def main(): + print("a cache miss fetches the window once") + cache, fetches = DictCache(), [] + recent = manager(sm.SoccerRecentManager, cache, fetches) + data = recent._fetch_soccer_api_data(use_cache=True) + check("one window fetch", len(fetches) == 1, fetches) + check("no stand-in 'partial' fetch of the same window", + "partial" not in fetches, fetches) + check("nothing submitted to the background service as well", + not recent.background_service.submit_fetch_request.called) + check("the fetched schedule is returned", data == {"events": [{"id": "1"}]}, data) + + print("\na cache hit fetches nothing") + again = recent._fetch_soccer_api_data(use_cache=True) + check("still one fetch", len(fetches) == 1, fetches) + check("the cached schedule is returned", again == data, again) + + print("\nthe recent and upcoming managers of a league share one fetch") + cache, fetches = DictCache(), [] + pair = [manager(sm.SoccerRecentManager, cache, fetches), + manager(sm.SoccerUpcomingManager, cache, fetches)] + results = [None, None] + + def run(i): + results[i] = pair[i]._fetch_soccer_api_data(use_cache=True) + + threads = [threading.Thread(target=run, args=(i,)) for i in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=10) + check("both missed together, one fetched", len(fetches) == 1, fetches) + check("both got the schedule", results[0] == results[1] == {"events": [{"id": "1"}]}, + results) + + print("\ndifferent leagues are not serialised behind each other") + cache, fetches = DictCache(), [] + leagues = ["eng.1", "esp.1", "ger.1", "ita.1"] + managers = [manager(sm.SoccerRecentManager, cache, fetches, league) for league in leagues] + started = time.monotonic() + threads = [threading.Thread(target=m._fetch_soccer_api_data) for m in managers] + for t in threads: + t.start() + for t in threads: + t.join(timeout=10) + elapsed = time.monotonic() - started + check("one fetch per league", len(fetches) == len(leagues), fetches) + check("they ran side by side (%.2fs for 4 x 0.05s)" % elapsed, elapsed < 0.15, elapsed) + + print("\nuse_cache=False still fetches") + cache, fetches = DictCache(), [] + m = manager(sm.SoccerRecentManager, cache, fetches) + m._fetch_soccer_api_data(use_cache=True) + m._fetch_soccer_api_data(use_cache=False) + check("two fetches", len(fetches) == 2, fetches) + + print("\nthe managers share the core's connection pool") + try: + from src.common.fetch_service import get_fetch_service + except ImportError: + print(" [skip] this core has no shared fetch service") + else: + check("the shared-pool helper was imported", + getattr(sm, "share_connection_pool", None) is not None) + import requests + from requests.adapters import HTTPAdapter + from urllib3.util.retry import Retry + + sessions = [] + for _ in range(2): + session = requests.Session() + session.mount("https://", HTTPAdapter(max_retries=Retry( + total=5, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["GET", "HEAD", "OPTIONS"]))) + sessions.append(session) + + # Run the constructor's sharing step on its own: the rest of the + # constructor needs fonts, a display and logos. + for session in sessions: + sm.share_connection_pool(session, session.get_adapter("https://").max_retries) + a, b = (s.get_adapter("https://") for s in sessions) + check("two managers' sessions use one adapter (one pool)", a is b) + check("...the core's shared one", a is get_fetch_service().shared_adapter(a.max_retries)) + check("...with the retry policy the manager mounted", + a.max_retries.total == 5 and a.max_retries.backoff_factor == 1) + src = (plugin_dir / "soccer_managers.py").read_text(encoding="utf-8") + check("BaseSoccerManager.__init__ shares its session's pool", + "share_connection_pool(\n self.session" in src) + + print("\n%s" % ("FAILED: %d" % len(failures) if failures else "All checks passed")) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())