Fix Backup Inventory 503 from unsupported Cosmos GROUP BY aggregate

application/single_app/config.py

@@ -96,7 +96,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.260.001"
+VERSION = "0.260.002"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')

application/single_app/functions_data_management.py

@@ -10538,6 +10538,22 @@ def get_data_management_backup_inventory(limit=100):
     ]
 
 
+def _count_data_management_backups(extra_clauses=None, extra_parameters=None):
+    """Count backups with a VALUE aggregate; the Python client cannot serve GROUP BY here."""
+    clauses = ["c.type = @type", "c.operation = @operation"]
+    parameters = [
+        {"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE},
+        {"name": "@operation", "value": DATA_MANAGEMENT_OPERATION_BACKUP},
+    ]
+    clauses.extend(extra_clauses or [])
+    parameters.extend(extra_parameters or [])
+    rows = _query_data_management_history_items(
+        query=f"SELECT VALUE COUNT(1) FROM c WHERE {' AND '.join(clauses)}",
+        parameters=parameters,
+    )
+    return _safe_int(rows[0], default=0, minimum=0) if rows else 0
+
+
 def _get_data_management_backup_global_summary():
     summary = {
         "full": 0,
@@ -10549,39 +10565,42 @@ def _get_data_management_backup_global_summary():
         "latest_full": None,
         "latest_partial": None,
     }
-    aggregate_query = (
-        "SELECT c.backup_type, c.status, COUNT(1) AS count FROM c "
-        "WHERE c.type = @type AND c.operation = @operation "
-        "GROUP BY c.backup_type, c.status"
-    )
     parameters = [
         {"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE},
         {"name": "@operation", "value": DATA_MANAGEMENT_OPERATION_BACKUP},
     ]
-    aggregate_rows = _query_data_management_history_items(
-        query=aggregate_query,
-        parameters=parameters,
+    available_clause = "(c.status = @completed OR c.status = @completed_with_warnings)"
+    available_parameters = [
+        {"name": "@completed", "value": DATA_MANAGEMENT_STATUS_COMPLETED},
+        {
+            "name": "@completed_with_warnings",
+            "value": DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS,
+        },
+    ]
+
+    summary["total"] = _count_data_management_backups()
+    summary["available"] = _count_data_management_backups(
+        [available_clause],
+        available_parameters,
     )
-    for row in aggregate_rows:
-        if not isinstance(row, dict):
-            continue
-        count = _safe_int(row.get("count"), default=0, minimum=0)
-        status = row.get("status")
-        backup_type = row.get("backup_type")
-        summary["total"] += count
-        if status in {
-            DATA_MANAGEMENT_STATUS_COMPLETED,
-            DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS,
-        }:
-            summary["available"] += count
-            if backup_type == DATA_MANAGEMENT_BACKUP_FULL:
-                summary["full"] += count
-            elif backup_type == DATA_MANAGEMENT_BACKUP_PARTIAL:
-                summary["partial"] += count
-        elif status == DATA_MANAGEMENT_STATUS_RUNNING:
-            summary["running"] += count
-        elif status == DATA_MANAGEMENT_STATUS_FAILED:
-            summary["failed"] += count
+    summary["running"] = _count_data_management_backups(
+        ["c.status = @running"],
+        [{"name": "@running", "value": DATA_MANAGEMENT_STATUS_RUNNING}],
+    )
+    summary["failed"] = _count_data_management_backups(
+        ["c.status = @failed"],
+        [{"name": "@failed", "value": DATA_MANAGEMENT_STATUS_FAILED}],
+    )
+    for backup_type, summary_field in (
+        (DATA_MANAGEMENT_BACKUP_FULL, "full"),
+        (DATA_MANAGEMENT_BACKUP_PARTIAL, "partial"),
+    ):
+        summary[summary_field] = _count_data_management_backups(
+            [available_clause, "c.backup_type = @backup_type"],
+            available_parameters + [
+                {"name": "@backup_type", "value": backup_type},
+            ],
+        )
 
     for backup_type, summary_field in (
         (DATA_MANAGEMENT_BACKUP_FULL, "latest_full"),

application/single_app/templates/admin_settings.html

@@ -6720,15 +6720,41 @@
                             <h4 class="mb-1">Backup Inventory</h4>
                             <p class="text-muted mb-0 small">Track completed full and partial backups created by Data Management jobs.</p>
                         </div>
-                        <div class="d-flex flex-wrap gap-2">
-                            <button type="button" class="btn btn-outline-danger btn-sm" id="data-management-run-retention-cleanup-btn">
+                        <div class="d-flex flex-wrap align-items-center gap-2">
+                            <button type="button" class="btn btn-outline-danger btn-sm" id="data-management-run-retention-cleanup-btn"
+                                    data-bs-toggle="tooltip" data-bs-placement="bottom"
+                                    title="Permanently deletes backups older than the configured retention period, along with their stored artifacts. Backups still inside the retention window are never touched.">
                                 <i class="bi bi-trash3 me-1"></i>Run Retention Cleanup
                             </button>
+                            <button type="button" class="btn btn-link btn-sm p-0 text-decoration-none" id="data-management-retention-cleanup-info-btn"
+                                    data-bs-toggle="collapse" data-bs-target="#data-management-retention-cleanup-help"
+                                    aria-expanded="false" aria-controls="data-management-retention-cleanup-help"
+                                    aria-label="What does Run Retention Cleanup do?">
+                                <i class="bi bi-info-circle" aria-hidden="true"></i>
+                            </button>
                             <button type="button" class="btn btn-outline-secondary btn-sm" id="data-management-refresh-backups-btn">
                                 <i class="bi bi-arrow-clockwise me-1"></i>Refresh
                             </button>
                         </div>
                     </div>
+                    <div class="collapse mb-3" id="data-management-retention-cleanup-help">
+                        <div class="alert alert-info small mb-0" role="note">
+                            <div class="fw-semibold mb-1"><i class="bi bi-info-circle me-1" aria-hidden="true"></i>What does Run Retention Cleanup do?</div>
+                            <p class="mb-2">
+                                It permanently deletes backups whose age exceeds the retention period configured in Data Management settings,
+                                and removes their stored artifacts from the backup container. Backups newer than the retention cutoff are left alone.
+                            </p>
+                            <ul class="mb-2 ps-3">
+                                <li>Only backups in a finished state are eligible; running or queued jobs are skipped.</li>
+                                <li>When <em>Keep latest full backup</em> is enabled, the most recent successful full backup is protected even if it is past the cutoff.</li>
+                                <li>Each run deletes at most 25 backups, so very large cleanups may need several runs.</li>
+                                <li>Cleanup also runs automatically on the configured schedule; this button just runs it now.</li>
+                            </ul>
+                            <p class="mb-0 text-muted">
+                                Seeing &ldquo;found no expired backups to delete&rdquo; means every backup is still inside the retention window. That is expected, not an error.
+                            </p>
+                        </div>
+                    </div>
                     <div class="row g-3 mb-3" role="group" aria-label="Backup inventory filters">
                         <div class="col-lg-4">
                             <button type="button" class="btn btn-primary w-100 h-100 text-start" id="data-management-view-all-backups-btn" data-backup-filter="all" aria-pressed="true">

docs/explanation/fixes/BACKUP_INVENTORY_GROUP_BY_AGGREGATE_FIX.md

@@ -0,0 +1,147 @@
+# Backup Inventory GROUP BY Aggregate Fix
+
+**Fixed in version: 0.260.002**
+
+## Issue Description
+
+The Backup Inventory panel in Admin Settings → Data Management always failed to load. The
+UI reported `Backup history is temporarily unavailable. Please try again.` and every summary
+tile rendered zeros:
+
+- Available backups: `0`
+- Full backups: `0`
+- Partial backups: `0`
+
+This happened even when backups had completed successfully, and even when the job list itself
+loaded correctly. The `GET /api/admin/data-management/backups` route returned `503`.
+
+## Root Cause
+
+`_get_data_management_backup_global_summary()` computed its counts with a single grouped
+aggregate:
+
+```sql
+SELECT c.backup_type, c.status, COUNT(1) AS count FROM c
+WHERE c.type = @type AND c.operation = @operation
+GROUP BY c.backup_type, c.status
+```
+
+The `azure-cosmos` Python client does not advertise support for `GroupBy` combined with a
+**non-VALUE aggregate** (`COUNT(1) AS count`). During query plan negotiation the service
+rejects the request:
+
+```
+(BadRequest) {"code":"BadRequest","message":"Query contains the following features,
+which the calling client does not support:\nNone GroupBy NonValueAggregate ..."}
+```
+
+The `BadRequest` propagated out of `_query_data_management_history_items`, was classified as
+a provider failure, and surfaced to the admin as a generic `503`.
+
+This was never a throttling, indexing, or capacity problem. Cosmos Maintenance consistently
+reported the composite index as **Aligned / 0 missing / 7 containers checked**, and the
+`400` entries with `requestCharge = 0` and sub-millisecond duration visible in
+`AzureDiagnostics` are the normal, benign cross-partition query-plan negotiation.
+
+Because the aggregate was part of the summary from the moment the feature shipped, Backup
+Inventory had never worked in any deployment.
+
+### Why tests did not catch it
+
+`FakeHistoryContainer` in `functional_tests/test_data_management_history_pagination.py`
+implemented `GROUP BY c.backup_type, c.status` in Python and returned grouped rows. The test
+double supported a query shape the real client cannot serve, so the suite passed against a
+query that always failed in production.
+
+## Technical Details
+
+### Files Modified
+
+| File | Change |
+|------|--------|
+| `application/single_app/functions_data_management.py` | Replaced the grouped aggregate with bounded `SELECT VALUE COUNT(1)` queries; added `_count_data_management_backups()` |
+| `application/single_app/templates/admin_settings.html` | Added retention cleanup tooltip and expandable help panel |
+| `functional_tests/test_data_management_history_pagination.py` | Fake now rejects `GROUP BY` / non-VALUE aggregates like the real client; added coverage |
+| `application/single_app/config.py` | Version bump |
+
+### Code Changes
+
+A new helper issues a single supported scalar aggregate:
+
+```python
+def _count_data_management_backups(extra_clauses=None, extra_parameters=None):
+    """Count backups with a VALUE aggregate; the Python client cannot serve GROUP BY here."""
+    clauses = ["c.type = @type", "c.operation = @operation"]
+    parameters = [...]
+    clauses.extend(extra_clauses or [])
+    parameters.extend(extra_parameters or [])
+    rows = _query_data_management_history_items(
+        query=f"SELECT VALUE COUNT(1) FROM c WHERE {' AND '.join(clauses)}",
+        parameters=parameters,
+    )
+    return _safe_int(rows[0], default=0, minimum=0) if rows else 0
+```
+
+The summary now issues six bounded counts — `total`, `available`, `running`, `failed`,
+`full`, and `partial`. `SELECT VALUE COUNT(1)` is fully supported by the Python client and is
+already used elsewhere in this module. The two `SELECT TOP 1 *` queries that resolve
+`latest_full` and `latest_partial` were already valid and are unchanged.
+
+This deliberately avoids the alternative of projecting every backup row and aggregating on the
+client, which would grow unbounded as backup history accumulates. Each count is served by
+Cosmos and returns a single scalar.
+
+### Retention Cleanup Guidance
+
+Admins had no way to tell what **Run Retention Cleanup** did, and a run that deleted nothing
+looked like a failure. The button now carries a hover tooltip, and an `(i)` toggle expands an
+inline panel explaining that cleanup:
+
+- permanently deletes backups older than the configured retention period, plus their stored artifacts;
+- only considers backups in a finished state;
+- protects the most recent successful full backup when **Keep latest full backup** is enabled;
+- deletes at most 25 backups per run;
+- also runs automatically on the configured schedule.
+
+The panel calls out that `found no expired backups to delete` means every backup is still
+inside the retention window, which is expected rather than an error.
+
+## Validation
+
+### Test Results
+
+`functional_tests/test_data_management_history_pagination.py` — **14 passed**.
+
+New and updated coverage:
+
+- `test_backup_summary_avoids_unsupported_group_by_aggregates` asserts no emitted history query
+  contains `GROUP BY` or `COUNT(1) AS`, and that exactly six `SELECT VALUE COUNT(1)` queries are issued.
+- `test_retention_cleanup_button_explains_what_it_does` asserts the tooltip, the `(i)` toggle,
+  the collapse target, and the explanatory copy are present.
+- `FakeHistoryContainer` now raises the real `BadRequest ... GroupBy NonValueAggregate` error
+  instead of emulating `GROUP BY`.
+
+### Regression Probe
+
+The fix was neutralized by restoring a grouped aggregate. Two tests failed with the exact
+production error:
+
+```
+RuntimeError: (BadRequest) Query contains the following features, which the calling client
+does not support: GroupBy NonValueAggregate.
+```
+
+- `test_backup_summary_is_global_and_page_independent`
+- `test_backup_summary_avoids_unsupported_group_by_aggregates`
+
+The fix was then restored and all 14 tests passed again, confirming the tests fail for the
+right reason.
+
+### Before / After
+
+| Behavior | Before | After |
+|----------|--------|-------|
+| `GET /api/admin/data-management/backups` | `503` | `200` |
+| Backup Inventory tiles | Always `0` | Actual counts |
+| Backup Inventory table | Error banner | Backup rows |
+| Retention cleanup purpose | Undocumented in UI | Tooltip + expandable help |

docs/explanation/release_notes.md

@@ -2,6 +2,25 @@
 
 For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/).
 
+### **(v0.260.002)**
+
+#### Bug Fixes
+
+*   **Backup Inventory No Longer Fails To Load**
+    *   Fixed the Backup Inventory panel in Admin Settings → Data Management always returning `503` and showing `0` for available, full, and partial backups.
+    *   The global summary used a Cosmos `GROUP BY` with a non-VALUE aggregate (`COUNT(1) AS count`), a combination the `azure-cosmos` Python client does not support. Cosmos rejected the query during plan negotiation with `BadRequest ... GroupBy NonValueAggregate`.
+    *   Counts are now computed with bounded, fully supported `SELECT VALUE COUNT(1)` queries, so the panel renders real numbers without loading backup history into memory.
+    *   This was not a throttling or indexing problem; the composite index was already aligned. Backup Inventory had been broken since the summary shipped.
+    *   (Ref: `functions_data_management.py`, `_get_data_management_backup_global_summary`, `_count_data_management_backups`, `/api/admin/data-management/backups`)
+
+#### User Interface Enhancements
+
+*   **Run Retention Cleanup Now Explains Itself**
+    *   Added a hover tooltip and an `(i)` toggle that expands inline guidance next to the **Run Retention Cleanup** button.
+    *   Documents that cleanup permanently deletes backups past the retention period along with their artifacts, skips jobs that are still running, honors **Keep latest full backup**, and deletes at most 25 backups per run.
+    *   Clarifies that "found no expired backups to delete" means every backup is still inside the retention window, which is expected rather than a failure.
+    *   (Ref: `admin_settings.html`, backup retention cleanup, Data Management)
+
 ### **(v0.260.001)**
 
 v0.260.001 consolidates all work released after v0.250.001 into one major release note, spanning 117 incremental patch builds. This rollup highlights the major feature, UI, reliability, security, and operations themes while preserving the full per-build history in the Detailed Change Log at the end of this section.