From 885f9d04c0681325de9b84550a0c401e6dd56305 Mon Sep 17 00:00:00 2001 From: ChuckBuilds Date: Thu, 1 Oct 2026 22:49:54 -0400 Subject: [PATCH 1/3] perf(sync): import numpy where send_frame uses it The web interface's API blueprint imports src.common.sync_manager only for STATUS_FILE and SYNC_PORT, which loaded numpy into the web process. numpy is used in one place, the leader's send_frame, so import it there. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uc5DAbSwGGUTm3MrCtpC2m --- src/common/sync_manager.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/common/sync_manager.py b/src/common/sync_manager.py index 2a2005251..ce43de344 100644 --- a/src/common/sync_manager.py +++ b/src/common/sync_manager.py @@ -29,7 +29,6 @@ import logging from enum import Enum from typing import Callable, Optional -import numpy as np from PIL import Image from src.config_manager_atomic import _replace @@ -434,6 +433,12 @@ def send_frame(self, image: Image.Image) -> None: return if self._leader_state != LeaderState.CONNECTED or not self._peer_ip: return + # numpy is imported here, not at module level: the web interface + # imports this module for its constants (STATUS_FILE, SYNC_PORT) and + # would otherwise load numpy for nothing. Only a connected leader + # gets this far, and after the first frame the import is a + # sys.modules lookup. + import numpy as np try: arr = np.asarray(image.convert("RGB"), dtype=np.uint8) header = _RAW_MAGIC + _RAW_HEADER.pack(image.width, image.height) From d3ef625d0c291062c9fa810de35935f911ac7960 Mon Sep 17 00:00:00 2001 From: ChuckBuilds Date: Thu, 1 Oct 2026 22:49:54 -0400 Subject: [PATCH 2/3] perf(common,plugin_system): import package re-exports on first use src/common/__init__.py and src/plugin_system/__init__.py imported every re-exported helper eagerly, so any submodule import -- the web interface's path_safety, store_manager, schema_manager -- also loaded ScrollHelper (numpy), LogoHelper, APIHelper, the adaptive layout helpers (freetype) and PluginManager. Resolve those names through a PEP 562 module __getattr__ instead, with __dir__, an unchanged __all__ and TYPE_CHECKING imports so mypy still sees the real types. Each name resolves to the same object as before, and is cached in the module namespace on first access. With the sync_manager change, importing web_interface.app on a Pi 4 drops from ~67 MB to ~54 MB RSS and no longer loads numpy. The display process ends up with the same modules loaded. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Uc5DAbSwGGUTm3MrCtpC2m --- CHANGELOG.md | 21 +++++ src/common/README.md | 4 +- src/common/__init__.py | 139 ++++++++++++++++++++++-------- src/plugin_system/__init__.py | 35 +++++++- test/test_lazy_package_imports.py | 120 ++++++++++++++++++++++++++ 5 files changed, 280 insertions(+), 39 deletions(-) create mode 100644 test/test_lazy_package_imports.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c9f698a..b3d418e85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,27 @@ accepts both, but the store flags the old spelling as deprecated `scripts/render_bench.py` records the same. Diagnostic only: nothing tunes, freezes or disables the collector. +### Web interface: lighter package imports + +- `src.common` and `src.plugin_system` now import their re-exported names on + first use (PEP 562 module `__getattr__`) instead of in `__init__.py`. + `from src.common import ScrollHelper`, `src.plugin_system.PluginManager`, + `from src.common import *` and every submodule import work as before and + return the same objects. What changes is that importing a submodule -- + the web interface's `src.common.path_safety`, `src.plugin_system.store_manager` + and the like -- no longer loads `ScrollHelper`, `LogoHelper`, `APIHelper`, + the adaptive layout helpers and `PluginManager` with it. `sync_manager` + imports numpy inside `send_frame`, the one place it uses it, since the API + blueprint imports that module only for its constants. +- The web process no longer loads numpy at all. On a Pi 4 (Python 3.13), + importing `web_interface.app` went from ~67 MB to ~54 MB RSS and from + ~2.8 s to ~1.3 s (`-X importtime`, median of five). A bare + `import src.common` went from ~50 MB / ~0.85 s to ~10 MB / ~30 ms. The + display process loads the same modules as before, only later. +- A misspelt name in `from src.common import ...` still raises `ImportError`. + `test/test_lazy_package_imports.py` checks that the packages import nothing + heavy and that every name in `__all__` resolves to its home module's object. + ### Outlined text: one rasterization - New `draw_text_outlined(draw, xy, text, font, fill, outline_color=(0, 0, diff --git a/src/common/README.md b/src/common/README.md index a2bcd29a2..9181db496 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -17,7 +17,9 @@ Rules for the package: - `from src.common import ...` re-exports `APIHelper`, `ScrollHelper`, `LogoHelper`, `TextHelper`, `scroll_config` (plus `ScrollSettings`, `configure_scroll`, `resolve_scroll_settings`, `refresh_hz_from_config`) and - the adaptive layout names below ([`__init__.py`](__init__.py)). + the adaptive layout names below ([`__init__.py`](__init__.py)). Each is + imported on first use, so `import src.common` or a submodule import stays + cheap; add a new re-export to `_LAZY` there as well as `__all__`. ## Summary diff --git a/src/common/__init__.py b/src/common/__init__.py index 9b3a89251..d22933afa 100644 --- a/src/common/__init__.py +++ b/src/common/__init__.py @@ -6,45 +6,90 @@ - Logo helpers - Text/scroll helpers - Adaptive layout and image helpers + +The names below are imported on first use (PEP 562), not when the package is +imported. ``from src.common import ScrollHelper`` and +``src.common.ScrollHelper`` work as before and return the same objects, but +``import src.common`` -- or importing any submodule, such as +``src.common.path_safety`` -- no longer loads numpy, requests and freetype +along with every helper. The web interface imports src.common only for a few +small modules and never needs those. """ -# Export commonly used utilities -from src.common.api_helper import APIHelper -from src.common.scroll_helper import ScrollHelper -from src.common import scroll_config -from src.common.scroll_config import ( - ScrollSettings, - configure as configure_scroll, - resolve as resolve_scroll_settings, - refresh_hz_from_config, -) -from src.common.logo_helper import LogoHelper -from src.common.text_helper import TextHelper +import importlib +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +if TYPE_CHECKING: + # What mypy and editors see: the real names and their types. + from src.common.api_helper import APIHelper + from src.common.scroll_helper import ScrollHelper + from src.common import scroll_config + from src.common.scroll_config import ( + ScrollSettings, + configure as configure_scroll, + resolve as resolve_scroll_settings, + refresh_hz_from_config, + ) + from src.common.logo_helper import LogoHelper + from src.common.text_helper import TextHelper -# Adaptive layout & images (canonical homes: src.adaptive_layout / -# src.adaptive_images — re-exported here so plugin authors find them in the -# blessed-helpers package). See docs/ADAPTIVE_LAYOUT.md. -from src.adaptive_layout import ( - Region, - LayoutContext, - FontStep, - FontLadder, - LADDER_GRID, - LADDER_ARCADE, - FitResult, - draw_fitted_text, - ScoreboardRegions, - scoreboard_regions, - MediaRow, - media_row, -) -from src.adaptive_images import ( - ImageFitResult, - fit_image, - draw_fitted_image, - RESAMPLE_LANCZOS, - RESAMPLE_NEAREST, -) + # Adaptive layout & images (canonical homes: src.adaptive_layout / + # src.adaptive_images — re-exported here so plugin authors find them in the + # blessed-helpers package). See docs/ADAPTIVE_LAYOUT.md. + from src.adaptive_layout import ( + Region, + LayoutContext, + FontStep, + FontLadder, + LADDER_GRID, + LADDER_ARCADE, + FitResult, + draw_fitted_text, + ScoreboardRegions, + scoreboard_regions, + MediaRow, + media_row, + ) + from src.adaptive_images import ( + ImageFitResult, + fit_image, + draw_fitted_image, + RESAMPLE_LANCZOS, + RESAMPLE_NEAREST, + ) + +#: Exported name -> (module it lives in, attribute name there). An attribute +#: of None means the name is the module itself. Keep in step with the +#: TYPE_CHECKING imports above and with __all__. +_LAZY: Dict[str, Tuple[str, Optional[str]]] = { + 'APIHelper': ('src.common.api_helper', 'APIHelper'), + 'ScrollHelper': ('src.common.scroll_helper', 'ScrollHelper'), + 'scroll_config': ('src.common.scroll_config', None), + 'ScrollSettings': ('src.common.scroll_config', 'ScrollSettings'), + 'configure_scroll': ('src.common.scroll_config', 'configure'), + 'resolve_scroll_settings': ('src.common.scroll_config', 'resolve'), + 'refresh_hz_from_config': ('src.common.scroll_config', 'refresh_hz_from_config'), + 'LogoHelper': ('src.common.logo_helper', 'LogoHelper'), + 'TextHelper': ('src.common.text_helper', 'TextHelper'), + # adaptive layout & images + 'Region': ('src.adaptive_layout', 'Region'), + 'LayoutContext': ('src.adaptive_layout', 'LayoutContext'), + 'FontStep': ('src.adaptive_layout', 'FontStep'), + 'FontLadder': ('src.adaptive_layout', 'FontLadder'), + 'LADDER_GRID': ('src.adaptive_layout', 'LADDER_GRID'), + 'LADDER_ARCADE': ('src.adaptive_layout', 'LADDER_ARCADE'), + 'FitResult': ('src.adaptive_layout', 'FitResult'), + 'draw_fitted_text': ('src.adaptive_layout', 'draw_fitted_text'), + 'ScoreboardRegions': ('src.adaptive_layout', 'ScoreboardRegions'), + 'scoreboard_regions': ('src.adaptive_layout', 'scoreboard_regions'), + 'MediaRow': ('src.adaptive_layout', 'MediaRow'), + 'media_row': ('src.adaptive_layout', 'media_row'), + 'ImageFitResult': ('src.adaptive_images', 'ImageFitResult'), + 'fit_image': ('src.adaptive_images', 'fit_image'), + 'draw_fitted_image': ('src.adaptive_images', 'draw_fitted_image'), + 'RESAMPLE_LANCZOS': ('src.adaptive_images', 'RESAMPLE_LANCZOS'), + 'RESAMPLE_NEAREST': ('src.adaptive_images', 'RESAMPLE_NEAREST'), +} __all__ = [ 'APIHelper', @@ -75,3 +120,25 @@ 'RESAMPLE_LANCZOS', 'RESAMPLE_NEAREST', ] + + +def __getattr__(name: str) -> Any: + """Import an exported name on first access (PEP 562). + + Only called for names not already in the module namespace, so after the + first access the cached value below is returned directly. Unknown names + raise AttributeError, which ``from src.common import `` relies + on to fall through to importing the submodule. + """ + try: + module_name, attr = _LAZY[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + module = importlib.import_module(module_name) + value = module if attr is None else getattr(module, attr) + globals()[name] = value + return value + + +def __dir__() -> List[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/src/plugin_system/__init__.py b/src/plugin_system/__init__.py index 9032c599a..ce88c359b 100644 --- a/src/plugin_system/__init__.py +++ b/src/plugin_system/__init__.py @@ -3,15 +3,46 @@ This module provides the core plugin infrastructure for the LEDMatrix project. It enables dynamic loading, management, and discovery of display plugins. + +BasePlugin and PluginManager are imported on first use (PEP 562), not when +the package is imported: the web interface imports several submodules +(store_manager, schema_manager, ...) and never needs PluginManager, which +pulls in the loader, executor and the shared helpers behind them. +``from src.plugin_system import BasePlugin`` works as before and returns the +same class. """ +import importlib +from typing import TYPE_CHECKING, Any, Dict, List, Tuple + __version__ = "1.0.0" -from .base_plugin import BasePlugin -from .plugin_manager import PluginManager +if TYPE_CHECKING: + from .base_plugin import BasePlugin + from .plugin_manager import PluginManager + +#: Exported name -> (module it lives in, attribute name there). +_LAZY: Dict[str, Tuple[str, str]] = { + 'BasePlugin': ('src.plugin_system.base_plugin', 'BasePlugin'), + 'PluginManager': ('src.plugin_system.plugin_manager', 'PluginManager'), +} __all__ = [ 'BasePlugin', 'PluginManager', ] + +def __getattr__(name: str) -> Any: + """Import an exported name on first access (PEP 562); see src.common.""" + try: + module_name, attr = _LAZY[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + value = getattr(importlib.import_module(module_name), attr) + globals()[name] = value + return value + + +def __dir__() -> List[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/test/test_lazy_package_imports.py b/test/test_lazy_package_imports.py new file mode 100644 index 000000000..28195fbaa --- /dev/null +++ b/test/test_lazy_package_imports.py @@ -0,0 +1,120 @@ +"""src.common and src.plugin_system import their exports lazily (PEP 562). + +The web interface imports both packages only for small submodules +(path_safety, snapshot_policy, store_manager, ...). When their __init__ +imported every export eagerly, that dragged numpy, freetype and the plugin +manager into a process that never uses them -- about 13 MB of RSS on a Pi. + +These tests pin both halves of the change: the package import stays light, +and every exported name still resolves to the very object its home module +defines, so ``isinstance`` and ``is`` checks behave as before. +""" + +import importlib +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + +#: Modules the bare package import must not load. numpy is the one that +#: matters for memory; the rest are what the eager __init__ used to import. +HEAVY = ( + "numpy", + "freetype", + "src.adaptive_layout", + "src.common.api_helper", + "src.common.logo_helper", + "src.common.scroll_helper", + "src.plugin_system.base_plugin", + "src.plugin_system.plugin_manager", +) + + +def _run(script): + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(script)], cwd=str(REPO_ROOT), + capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"]) +def test_package_import_loads_nothing_heavy(package): + # A fresh interpreter: this test process has long since imported them. + loaded = _run(f""" + import json, sys + import {package} + print(json.dumps(sorted(m for m in {HEAVY!r} if m in sys.modules))) + """) + assert loaded == [], f"importing {package} loaded {loaded}" + + +def test_web_interface_submodules_do_not_load_numpy(): + # The imports web_interface/app.py and its API blueprint make from these + # packages. numpy here costs the web process ~13 MB for nothing. + loaded = _run(""" + import json, sys + from src.common import path_safety, snapshot_policy, sync_manager + from src.plugin_system import store_manager, schema_manager + print(json.dumps("numpy" in sys.modules)) + """) + assert loaded is False + + +@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"]) +def test_every_exported_name_resolves_to_its_home_object(package): + pkg = importlib.import_module(package) + assert sorted(pkg.__all__) == sorted(pkg._LAZY), "__all__ and _LAZY differ" + for name in pkg.__all__: + module_name, attr = pkg._LAZY[name] + home = importlib.import_module(module_name) + expected = home if attr is None else getattr(home, attr) + assert getattr(pkg, name) is expected, name + assert name in dir(pkg) + + +def test_from_import_forms_plugins_use(): + # Every form found in ledmatrix-plugins and core: names, aliases, + # submodules through the package, dotted submodule imports. + from src.common import ScrollHelper, LogoHelper + from src.common import scroll_config as _scroll_config + from src.common import sports_card as _card + from src.common import draw_fitted_text + from src.plugin_system import BasePlugin, PluginManager + from src.plugin_system import compatibility + import src.common.scroll_helper + import src.plugin_system.base_plugin + import src.common + import src.plugin_system + + assert ScrollHelper is src.common.scroll_helper.ScrollHelper + assert LogoHelper is src.common.LogoHelper + assert _scroll_config is src.common.scroll_config + assert _card is importlib.import_module("src.common.sports_card") + assert draw_fitted_text is importlib.import_module("src.adaptive_layout").draw_fitted_text + assert BasePlugin is src.plugin_system.base_plugin.BasePlugin + assert PluginManager is importlib.import_module( + "src.plugin_system.plugin_manager").PluginManager + assert compatibility is importlib.import_module("src.plugin_system.compatibility") + assert src.plugin_system.__version__ == "1.0.0" + + +def test_star_import_still_binds_everything(): + namespace = {} + exec("from src.common import *", namespace) + import src.common + assert set(src.common.__all__) <= set(namespace) + + +@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"]) +def test_unknown_name_raises_attribute_error(package): + pkg = importlib.import_module(package) + with pytest.raises(AttributeError, match="no_such_name"): + pkg.no_such_name # noqa: B018 + with pytest.raises(ImportError): + exec(f"from {package} import no_such_name") From 0994a9a18ae1b376f2b8b7695ac8ae6677ceb5ee Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:24:08 -0400 Subject: [PATCH 3/3] chore(imports): mark the lazy re-export imports as a fixed table for Semgrep Codacy flagged importlib.import_module(module_name) as a non-literal import; module_name only ever comes from the module's own _LAZY map. Co-Authored-By: Claude Opus 5.5 --- src/common/__init__.py | 2 +- src/plugin_system/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/__init__.py b/src/common/__init__.py index d22933afa..b6409bde2 100644 --- a/src/common/__init__.py +++ b/src/common/__init__.py @@ -134,7 +134,7 @@ def __getattr__(name: str) -> Any: module_name, attr = _LAZY[name] except KeyError: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None - module = importlib.import_module(module_name) + module = importlib.import_module(module_name) # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import -- module_name comes from the fixed _LAZY table value = module if attr is None else getattr(module, attr) globals()[name] = value return value diff --git a/src/plugin_system/__init__.py b/src/plugin_system/__init__.py index ce88c359b..79eb03134 100644 --- a/src/plugin_system/__init__.py +++ b/src/plugin_system/__init__.py @@ -39,7 +39,7 @@ def __getattr__(name: str) -> Any: module_name, attr = _LAZY[name] except KeyError: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None - value = getattr(importlib.import_module(module_name), attr) + value = getattr(importlib.import_module(module_name), attr) # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import -- module_name comes from the fixed _LAZY table globals()[name] = value return value