From 0b065893bbc8c64e8d545df69bba52beeb736bf4 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 19 Aug 2026 09:02:46 -0400 Subject: [PATCH] Fix Backup Inventory 503 from unsupported Cosmos GROUP BY aggregate Backup Inventory always failed to load, returning 503 and rendering 0 for available, full, and partial backups even when backups had completed. _get_data_management_backup_global_summary() computed its counts with "GROUP BY c.backup_type, c.status" combined with a non-VALUE aggregate (COUNT(1) AS count). The azure-cosmos Python client does not advertise support for that combination, so Cosmos rejects it during query plan negotiation with BadRequest "GroupBy NonValueAggregate", which surfaced to admins as a generic 503. This was not throttling or a missing index; the composite index was already aligned. The aggregate shipped with the summary feature, so Backup Inventory had never worked in any deployment. Replace the grouped aggregate with six bounded SELECT VALUE COUNT(1) queries via a new _count_data_management_backups() helper. VALUE aggregates are supported by the client and already used elsewhere in this module. This avoids projecting every backup row and aggregating client-side, which would grow unbounded as history accumulates. The SELECT TOP 1 latest-backup queries were already valid and are unchanged. FakeHistoryContainer previously implemented GROUP BY in Python, so the suite passed against a query that always failed in production. It now raises the real BadRequest error instead, and a new test asserts no emitted history query uses GROUP BY or a non-VALUE aggregate. Also add a hover tooltip and an (i) toggle explaining what Run Retention Cleanup does, including that "found no expired backups to delete" means every backup is still inside the retention window rather than an error. Validation: 14 passed in the history pagination suite; 159 passed across Data Management with only the known pre-existing durability failure. Regression probe restoring a grouped aggregate fails two tests with the exact production error. Fixes #1294 --- application/single_app/config.py | 2 +- .../single_app/functions_data_management.py | 75 +++++---- .../single_app/templates/admin_settings.html | 30 +++- ...BACKUP_INVENTORY_GROUP_BY_AGGREGATE_FIX.md | 147 ++++++++++++++++++ docs/explanation/release_notes.md | 19 +++ ...test_data_management_history_pagination.py | 70 +++++++-- 6 files changed, 299 insertions(+), 44 deletions(-) create mode 100644 docs/explanation/fixes/BACKUP_INVENTORY_GROUP_BY_AGGREGATE_FIX.md diff --git a/application/single_app/config.py b/application/single_app/config.py index 9dabbc5a3..26c151c66 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.260.001" +VERSION = "0.260.002" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_data_management.py b/application/single_app/functions_data_management.py index 5b6658aef..0c8565bbb 100644 --- a/application/single_app/functions_data_management.py +++ b/application/single_app/functions_data_management.py @@ -10538,6 +10538,22 @@ def get_data_management_backup_inventory(limit=100): ] +def _count_data_management_backups(extra_clauses=None, extra_parameters=None): + """Count backups with a VALUE aggregate; the Python client cannot serve GROUP BY here.""" + clauses = ["c.type = @type", "c.operation = @operation"] + parameters = [ + {"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE}, + {"name": "@operation", "value": DATA_MANAGEMENT_OPERATION_BACKUP}, + ] + clauses.extend(extra_clauses or []) + parameters.extend(extra_parameters or []) + rows = _query_data_management_history_items( + query=f"SELECT VALUE COUNT(1) FROM c WHERE {' AND '.join(clauses)}", + parameters=parameters, + ) + return _safe_int(rows[0], default=0, minimum=0) if rows else 0 + + def _get_data_management_backup_global_summary(): summary = { "full": 0, @@ -10549,39 +10565,42 @@ def _get_data_management_backup_global_summary(): "latest_full": None, "latest_partial": None, } - aggregate_query = ( - "SELECT c.backup_type, c.status, COUNT(1) AS count FROM c " - "WHERE c.type = @type AND c.operation = @operation " - "GROUP BY c.backup_type, c.status" - ) parameters = [ {"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE}, {"name": "@operation", "value": DATA_MANAGEMENT_OPERATION_BACKUP}, ] - aggregate_rows = _query_data_management_history_items( - query=aggregate_query, - parameters=parameters, + available_clause = "(c.status = @completed OR c.status = @completed_with_warnings)" + available_parameters = [ + {"name": "@completed", "value": DATA_MANAGEMENT_STATUS_COMPLETED}, + { + "name": "@completed_with_warnings", + "value": DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS, + }, + ] + + summary["total"] = _count_data_management_backups() + summary["available"] = _count_data_management_backups( + [available_clause], + available_parameters, ) - for row in aggregate_rows: - if not isinstance(row, dict): - continue - count = _safe_int(row.get("count"), default=0, minimum=0) - status = row.get("status") - backup_type = row.get("backup_type") - summary["total"] += count - if status in { - DATA_MANAGEMENT_STATUS_COMPLETED, - DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS, - }: - summary["available"] += count - if backup_type == DATA_MANAGEMENT_BACKUP_FULL: - summary["full"] += count - elif backup_type == DATA_MANAGEMENT_BACKUP_PARTIAL: - summary["partial"] += count - elif status == DATA_MANAGEMENT_STATUS_RUNNING: - summary["running"] += count - elif status == DATA_MANAGEMENT_STATUS_FAILED: - summary["failed"] += count + summary["running"] = _count_data_management_backups( + ["c.status = @running"], + [{"name": "@running", "value": DATA_MANAGEMENT_STATUS_RUNNING}], + ) + summary["failed"] = _count_data_management_backups( + ["c.status = @failed"], + [{"name": "@failed", "value": DATA_MANAGEMENT_STATUS_FAILED}], + ) + for backup_type, summary_field in ( + (DATA_MANAGEMENT_BACKUP_FULL, "full"), + (DATA_MANAGEMENT_BACKUP_PARTIAL, "partial"), + ): + summary[summary_field] = _count_data_management_backups( + [available_clause, "c.backup_type = @backup_type"], + available_parameters + [ + {"name": "@backup_type", "value": backup_type}, + ], + ) for backup_type, summary_field in ( (DATA_MANAGEMENT_BACKUP_FULL, "latest_full"), diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index 8b15e6607..7bcd8410f 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -6720,15 +6720,41 @@

Cosmos DB JSON E

Backup Inventory

Track completed full and partial backups created by Data Management jobs.

-
- +
+
+
+
What does Run Retention Cleanup do?
+

+ It permanently deletes backups whose age exceeds the retention period configured in Data Management settings, + and removes their stored artifacts from the backup container. Backups newer than the retention cutoff are left alone. +

+
    +
  • Only backups in a finished state are eligible; running or queued jobs are skipped.
    • +
  • When Keep latest full backup is enabled, the most recent successful full backup is protected even if it is past the cutoff.
    • +
  • Each run deletes at most 25 backups, so very large cleanups may need several runs.
    • +
  • Cleanup also runs automatically on the configured schedule; this button just runs it now.
    • +
+

+ Seeing “found no expired backups to delete” means every backup is still inside the retention window. That is expected, not an error. +

+
+