feat(cache): add favorites cache warm-up strategy

docs/admin_docs/configuration/cache.mdx

@@ -282,6 +282,48 @@ Requirements and limitations:
 - Cascading/dependent native filters and search-term variants of filter option queries are not
   warmed by this strategy.
 
+### Warming Up Favorited Dashboards
+
+The `favorites` strategy warms the cache for each user's favorited dashboards. It is off by
+default; enable it and schedule it with Celery beat:
+
+```python
+CACHE_WARMUP_FAVORITES = True
+CACHE_WARMUP_FAVORITES_MAX_DASHBOARDS = 5  # most recently favorited dashboards per user
+CACHE_WARMUP_FAVORITES_QUEUE = "cache_warmup"  # optional dedicated Celery queue
+
+class CeleryConfig(CeleryConfig):
+    beat_schedule = {
+        "cache-warmup-favorites": {
+            "task": "cache-warmup",
+            "schedule": crontab(minute=0, hour="*/6"),
+            "kwargs": {
+                "strategy_name": "favorites",
+                "since": "7 days ago",  # only users who logged in since then
+                "max_users": 100,  # users handled per run, most recent login first
+            },
+        },
+    }
+```
+
+Each run enqueues one task per selected user. The task runs as that user, so the cache entries
+match what that user's dashboards request, and only charts whose query results are missing or
+expired are computed. It does not use `SUPERSET_CACHE_WARMUP_USER`.
+
+Notes:
+
+- A Celery worker and a real `DATA_CACHE_CONFIG` backend are required (with `NullCache` every
+  run would recompute).
+- A per-result lock, kept in `CACHE_CONFIG`, stops concurrent runs from computing the same
+  result twice; use a shared backend such as Redis for it to work across processes.
+- Set `CACHE_WARMUP_FAVORITES_QUEUE` and run a worker for that queue to keep warm-up from
+  competing with reports, alerts and thumbnails. Each per-user task has a 15 minute time limit.
+- Dashboard default column filters (native filter defaults and legacy defaults) are applied.
+  Other dashboard filter settings, such as time range, time column and grain, extras and ad hoc
+  filters, are not, so charts whose dashboard load uses them may still miss the cache.
+- `CACHE_WARMUP_FAVORITES` is an ordinary configuration setting, not a feature flag.
+- Queued per-user tasks expire after two hours if no worker picks them up.
+
 ## Caching Thumbnails
 
 This is an optional feature that can be turned on by activating its [feature flag](/admin-docs/configuration/configuring-superset#feature-flags) on config:

superset/commands/chart/warm_up_cache.py

@@ -18,6 +18,8 @@
 
 from typing import Any, cast, Optional, Union
 
+from celery.exceptions import SoftTimeLimitExceeded
+
 from superset.commands.base import BaseCommand
 from superset.commands.chart.data.get_data_command import ChartDataCommand
 from superset.commands.chart.exceptions import (
@@ -26,8 +28,9 @@
     WarmUpCacheChartNotFoundError,
 )
 from superset.common.db_query_status import QueryStatus
+from superset.constants import CACHE_DISABLED_TIMEOUT
 from superset.exceptions import SupersetSecurityException
-from superset.extensions import db, security_manager
+from superset.extensions import cache_manager, db, security_manager
 from superset.models.slice import Slice
 from superset.semantic_layers.access import (
     is_semantic_layers_enabled,
@@ -37,17 +40,24 @@
 from superset.utils.core import error_msg_from_exception, QueryObjectFilterClause
 from superset.views.utils import get_dashboard_extra_filters
 
+# Seconds a per-result warm-up lock is held if the worker dies before releasing it.
+WARMUP_LOCK_TTL = 900
+
 
 class ChartWarmUpCacheCommand(BaseCommand):
     def __init__(
         self,
         chart_or_id: Union[int, Slice],
         dashboard_id: Optional[int],
         extra_filters: Optional[str],
+        only_if_stale: bool = False,
     ):
         self._chart_or_id = chart_or_id
         self._dashboard_id = dashboard_id
         self._extra_filters = extra_filters
+        # When set, charts whose query results are all cached are left untouched
+        # and missing results are computed without bypassing the cache.
+        self._only_if_stale = only_if_stale
 
     def _get_dashboard_filters(self, chart_id: int) -> list[dict[str, Any]]:
         """Retrieve dashboard filters from extra_filters or dashboard metadata."""
@@ -59,6 +69,42 @@ def _get_dashboard_filters(self, chart_id: int) -> list[dict[str, Any]]:
 
         return get_dashboard_extra_filters(chart_id, self._dashboard_id)
 
+    @staticmethod
+    def _missing_cache_keys(query_context: Any) -> list[str]:
+        """Return cache keys of the context's queries lacking an unexpired entry."""
+        if query_context.get_cache_timeout() == CACHE_DISABLED_TIMEOUT:
+            # Results are never stored, so there is nothing to warm.
+            return []
+        missing: list[str] = []
+        for query in query_context.queries:
+            # Validation normalizes filters, matching the key used when querying.
+            query.validate()
+            cache_key = query_context.query_cache_key(query)
+            if not cache_key or cache_manager.data_cache.get(cache_key) is None:
+                missing.append(cache_key or "")
+        return missing
+
+    def _acquire_warmup_lock(self, query_context: Any) -> tuple[bool, str | None]:
+        """
+        Decide whether this worker should compute the chart's missing results.
+
+        Returns ``(should_run, lock_key)``. ``should_run`` is False when everything is
+        cached or another worker holds the lock; otherwise the returned lock key must
+        be released after the computation.
+        """
+        missing = self._missing_cache_keys(query_context)
+        if not missing:
+            return False, None
+        # Only one worker warms a given result at a time.
+        lock_key = f"cache-warmup-lock:{missing[0]}"
+        if not cache_manager.cache.add(lock_key, 1, timeout=WARMUP_LOCK_TTL):
+            return False, None
+        if not self._missing_cache_keys(query_context):
+            # Another worker stored the result while this one waited.
+            cache_manager.cache.delete(lock_key)
+            return False, None
+        return True, lock_key
+
     def _warm_up_non_legacy_cache(self, chart: Slice) -> tuple[Any, Any]:
         """Warm up cache for non-legacy visualizations."""
         query_context = chart.get_query_context()
@@ -77,10 +123,20 @@ def _warm_up_non_legacy_cache(self, chart: Slice) -> tuple[Any, Any]:
                     + query.filter
                 )
 
-        query_context.force = True
-        command = ChartDataCommand(query_context)
-        command.validate()
-        payload = command.run()
+        lock_key: str | None = None
+        if self._only_if_stale:
+            should_run, lock_key = self._acquire_warmup_lock(query_context)
+            if not should_run:
+                return None, QueryStatus.SUCCESS
+        else:
+            query_context.force = True
+        try:
+            command = ChartDataCommand(query_context)
+            command.validate()
+            payload = command.run()
+        finally:
+            if lock_key:
+                cache_manager.cache.delete(lock_key)
 
         # Report the first error.
         for query_result in cast(list[dict[str, Any]], payload["queries"]):
@@ -108,6 +164,8 @@ def run(self, *, skip_disabled: bool = False) -> dict[str, Any]:
 
         try:
             error, status = self._warm_up_non_legacy_cache(chart)
+        except SoftTimeLimitExceeded:
+            raise
         except Exception as ex:  # pylint: disable=broad-except
             error = error_msg_from_exception(ex)
             status = None

superset/config.py

@@ -1476,6 +1476,21 @@ def sync_theme_logo_href(
 # - Cascade/dependent filters and search-term variants are not warmed in this
 #   version; only base option queries are supported.
 
+# Gate for the `favorites` cache warm-up strategy, which warms charts on each
+# user's favorited dashboards. Schedule it through the `cache-warmup` Celery beat
+# task (see the `FavoritesStrategy` docstring in superset/tasks/cache.py). A
+# separate task per user runs as that user and computes only query results that
+# are missing or expired in DATA_CACHE_CONFIG. Requires a Celery worker and a real
+# DATA_CACHE_CONFIG backend (with NullCache every run would recompute).
+CACHE_WARMUP_FAVORITES = False
+# Maximum number of favorited dashboards (most recently favorited first) warmed
+# per user per run.
+CACHE_WARMUP_FAVORITES_MAX_DASHBOARDS = 5
+# Celery queue for per-user favorites warm-up tasks. Point it at a dedicated,
+# low-priority queue (served by its own worker) so a warm-up burst cannot delay
+# other tasks. None uses the default queue.
+CACHE_WARMUP_FAVORITES_QUEUE: str | None = None
+
 # Time before selenium times out after trying to locate an element on the page and wait
 # for that element to load for a screenshot.
 SCREENSHOT_LOCATE_WAIT = int(timedelta(seconds=10).total_seconds())