diff --git a/application/single_app/config.py b/application/single_app/config.py index 1d44c7ff5..1d6b6bacf 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -95,7 +95,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.103" +VERSION = "0.250.104" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_cosmos_indexing.py b/application/single_app/functions_cosmos_indexing.py index a55dc066f..f241e99f6 100644 --- a/application/single_app/functions_cosmos_indexing.py +++ b/application/single_app/functions_cosmos_indexing.py @@ -13,6 +13,8 @@ cosmos_collaboration_messages_container_name, cosmos_conversations_container, cosmos_conversations_container_name, + cosmos_data_management_jobs_container, + cosmos_data_management_jobs_container_name, cosmos_database, cosmos_group_documents_container, cosmos_group_documents_container_name, @@ -26,7 +28,7 @@ from functions_appinsights import log_event -COSMOS_INDEXING_POLICY_DEFINITION_VERSION = 1 +COSMOS_INDEXING_POLICY_DEFINITION_VERSION = 2 COSMOS_INDEXING_POLICY_APPLY_SETTING = 'app_maintenance_apply_cosmos_indexing_policies' COSMOS_INDEXING_POLICY_MAX_REPLACE_RETRIES = 3 @@ -80,6 +82,17 @@ def _composite_index(*paths): ], }, }, + { + 'container_name': cosmos_data_management_jobs_container_name, + 'container': cosmos_data_management_jobs_container, + 'partition_key_path': '/id', + 'description': 'Deterministic Data Management job and backup history pagination.', + 'expected_policy': { + 'compositeIndexes': [ + _composite_index(('/created_at', 'descending'), ('/id', 'descending')), + ], + }, + }, { 'container_name': cosmos_collaboration_messages_container_name, 'container': cosmos_collaboration_messages_container, diff --git a/application/single_app/functions_data_management.py b/application/single_app/functions_data_management.py index 5b6681656..ebb9fc51c 100644 --- a/application/single_app/functions_data_management.py +++ b/application/single_app/functions_data_management.py @@ -5,6 +5,7 @@ import base64 from email.utils import parsedate_to_datetime import hashlib +import hmac import io import json import logging @@ -37,7 +38,7 @@ from azure.search.documents.indexes import SearchIndexClient from azure.search.documents.indexes.models import SearchField, SearchFieldDataType, SearchIndex from azure.storage.blob import BlobBlock, BlobServiceClient, ContentSettings -from cryptography.fernet import Fernet +from cryptography.fernet import Fernet, InvalidToken import config as app_config from config import ( @@ -164,6 +165,11 @@ DATA_MANAGEMENT_DEFAULT_LEASE_SECONDS = 900 DATA_MANAGEMENT_DEFAULT_STALE_SECONDS = 1200 DATA_MANAGEMENT_DEFAULT_JOB_LIMIT = 25 +DATA_MANAGEMENT_HISTORY_MAX_PAGE_SIZE = 100 +DATA_MANAGEMENT_HISTORY_TOKEN_TTL_SECONDS = 3600 +DATA_MANAGEMENT_HISTORY_TOKEN_VERSION = 1 +DATA_MANAGEMENT_HISTORY_MAX_DATE_RANGE_DAYS = 366 +DATA_MANAGEMENT_HISTORY_SORT = "created_at_desc_id_desc" DATA_MANAGEMENT_DEFAULT_RECOVERY_JOB_LIMIT = 25 DATA_MANAGEMENT_RECOVERY_QUEUE_DELAY_SECONDS = 60 DATA_MANAGEMENT_RECOVERY_RESUBMIT_DELAY_SECONDS = 120 @@ -302,6 +308,10 @@ class DataManagementCosmosEditorError(ValueError): """Raised when Cosmos editor input is unsafe or incomplete.""" +class DataManagementHistoryPaginationError(ValueError): + """Raised when Data Management history pagination input is invalid.""" + + class DataManagementMigrationLeaseLostError(RuntimeError): """Raised when a stale migration worker no longer owns its job lease.""" @@ -7792,16 +7802,315 @@ def queue_data_management_job(operation, backup_type=None, requested_by=None, re raise -def get_data_management_jobs(limit=DATA_MANAGEMENT_DEFAULT_JOB_LIMIT): - safe_limit = _safe_int(limit, default=DATA_MANAGEMENT_DEFAULT_JOB_LIMIT, minimum=1, maximum=100) - query = "SELECT * FROM c WHERE c.type = @type ORDER BY c.created_at DESC" - parameters = [{"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE}] - return list(cosmos_data_management_jobs_container.query_items( +def _normalize_data_management_history_datetime(value, field_name, end_exclusive=False): + text = _safe_text(value) + if not text: + return None + date_only = bool(re.fullmatch(r"\d{4}-\d{2}-\d{2}", text)) + parsed = _parse_iso_datetime(text) + if parsed is None: + raise DataManagementHistoryPaginationError( + f"{field_name} must be an ISO-8601 date or timestamp." + ) + if date_only and end_exclusive: + parsed += timedelta(days=1) + return parsed.isoformat() + + +def normalize_data_management_history_filters(list_kind, filters=None): + normalized_kind = _safe_text(list_kind).lower() + if normalized_kind not in {"jobs", "backups"}: + raise DataManagementHistoryPaginationError("Unsupported Data Management history list.") + + raw_filters = filters if isinstance(filters, dict) else {} + status = _safe_text(raw_filters.get("status")).lower() + allowed_statuses = { + DATA_MANAGEMENT_STATUS_QUEUED, + DATA_MANAGEMENT_STATUS_RUNNING, + DATA_MANAGEMENT_STATUS_COMPLETED, + DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS, + DATA_MANAGEMENT_STATUS_FAILED, + DATA_MANAGEMENT_STATUS_CANCELED, + } + if normalized_kind == "backups": + allowed_statuses.add("available") + if status and status not in allowed_statuses: + raise DataManagementHistoryPaginationError("Unsupported Data Management status filter.") + + scheduled_value = _safe_text(raw_filters.get("scheduled")).lower() + if scheduled_value in {"", "all"}: + scheduled = None + elif scheduled_value == "scheduled": + scheduled = True + elif scheduled_value == "manual": + scheduled = False + else: + raise DataManagementHistoryPaginationError("Scheduled filter must be all, scheduled, or manual.") + + created_from = _normalize_data_management_history_datetime( + raw_filters.get("created_from"), + "created_from", + ) + created_to = _normalize_data_management_history_datetime( + raw_filters.get("created_to"), + "created_to", + end_exclusive=True, + ) + if bool(created_from) != bool(created_to): + raise DataManagementHistoryPaginationError( + "created_from and created_to must be provided together." + ) + if created_from and created_to: + start = _parse_iso_datetime(created_from) + end = _parse_iso_datetime(created_to) + if end <= start: + raise DataManagementHistoryPaginationError("created_to must be after created_from.") + if end - start > timedelta(days=DATA_MANAGEMENT_HISTORY_MAX_DATE_RANGE_DAYS): + raise DataManagementHistoryPaginationError( + f"Date range cannot exceed {DATA_MANAGEMENT_HISTORY_MAX_DATE_RANGE_DAYS} days." + ) + + operation = None + backup_type = None + if normalized_kind == "jobs": + operation = _safe_text(raw_filters.get("operation")).lower() + if operation and operation not in DATA_MANAGEMENT_OPERATIONS: + raise DataManagementHistoryPaginationError("Unsupported Data Management operation filter.") + else: + backup_type = _safe_text(raw_filters.get("backup_type")).lower() + if backup_type and backup_type not in DATA_MANAGEMENT_BACKUP_TYPES: + raise DataManagementHistoryPaginationError("Backup type filter must be full or partial.") + + return { + "operation": operation or None, + "backup_type": backup_type or None, + "status": status or None, + "scheduled": scheduled, + "created_from": created_from, + "created_to": created_to, + } + + +def _get_data_management_history_cipher(): + secret = _safe_text(getattr(app_config, "SECRET_KEY", "")) + derived_key = hashlib.sha256( + f"data-management-history:{secret}".encode("utf-8") + ).digest() + return Fernet(base64.urlsafe_b64encode(derived_key)) + + +def _encode_data_management_history_token(list_kind, filters, page_size, cursor): + if not isinstance(cursor, dict): + return None + cursor_created_at = _safe_text(cursor.get("created_at")) + cursor_id = _safe_text(cursor.get("id")) + if not cursor_created_at or not cursor_id: + return None + payload = { + "version": DATA_MANAGEMENT_HISTORY_TOKEN_VERSION, + "list_kind": list_kind, + "filters": filters, + "page_size": page_size, + "sort": DATA_MANAGEMENT_HISTORY_SORT, + "cursor": { + "created_at": cursor_created_at, + "id": cursor_id, + }, + } + token = _get_data_management_history_cipher().encrypt( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ) + return token.decode("utf-8") + + +def _decode_data_management_history_token(token, list_kind, filters, page_size): + safe_token = _safe_text(token) + if not safe_token: + return None + try: + decoded_token = base64.b64decode( + safe_token.encode("utf-8"), + altchars=b"-_", + validate=True, + ) + canonical_token = base64.urlsafe_b64encode(decoded_token).decode("utf-8") + if not hmac.compare_digest(canonical_token, safe_token): + raise ValueError("Continuation token is not canonical.") + decrypted = _get_data_management_history_cipher().decrypt( + safe_token.encode("utf-8"), + ttl=DATA_MANAGEMENT_HISTORY_TOKEN_TTL_SECONDS, + ) + payload = json.loads(decrypted.decode("utf-8")) + except (InvalidToken, UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise DataManagementHistoryPaginationError( + "Continuation token is invalid or expired." + ) from exc + + if ( + not isinstance(payload, dict) + or payload.get("version") != DATA_MANAGEMENT_HISTORY_TOKEN_VERSION + or payload.get("list_kind") != list_kind + or payload.get("filters") != filters + or payload.get("page_size") != page_size + or payload.get("sort") != DATA_MANAGEMENT_HISTORY_SORT + or not isinstance(payload.get("cursor"), dict) + or not _safe_text(payload["cursor"].get("created_at")) + or not _safe_text(payload["cursor"].get("id")) + ): + raise DataManagementHistoryPaginationError( + "Continuation token is invalid or does not match the active filters." + ) + return { + "created_at": _safe_text(payload["cursor"].get("created_at")), + "id": _safe_text(payload["cursor"].get("id")), + } + + +def _build_data_management_history_query(list_kind, filters, page_limit, cursor=None): + clauses = ["c.type = @type"] + parameters = [ + {"name": "@page_limit", "value": page_limit}, + {"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE}, + ] + if list_kind == "backups": + clauses.append("c.operation = @backup_operation") + parameters.append({ + "name": "@backup_operation", + "value": DATA_MANAGEMENT_OPERATION_BACKUP, + }) + elif filters.get("operation"): + clauses.append("c.operation = @operation") + parameters.append({"name": "@operation", "value": filters["operation"]}) + if filters.get("backup_type"): + clauses.append("c.backup_type = @backup_type") + parameters.append({"name": "@backup_type", "value": filters["backup_type"]}) + if filters.get("status") == "available": + clauses.append( + "(c.status = @completed OR c.status = @completed_with_warnings)" + ) + parameters.extend([ + {"name": "@completed", "value": DATA_MANAGEMENT_STATUS_COMPLETED}, + { + "name": "@completed_with_warnings", + "value": DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS, + }, + ]) + elif filters.get("status"): + clauses.append("c.status = @status") + parameters.append({"name": "@status", "value": filters["status"]}) + if filters.get("scheduled") is not None: + clauses.append("c.scheduled = @scheduled") + parameters.append({"name": "@scheduled", "value": filters["scheduled"]}) + if filters.get("created_from"): + clauses.append("c.created_at >= @created_from") + parameters.append({"name": "@created_from", "value": filters["created_from"]}) + if filters.get("created_to"): + clauses.append("c.created_at < @created_to") + parameters.append({"name": "@created_to", "value": filters["created_to"]}) + if isinstance(cursor, dict): + clauses.append( + "(c.created_at < @cursor_created_at " + "OR (c.created_at = @cursor_created_at AND c.id < @cursor_id))" + ) + parameters.extend([ + { + "name": "@cursor_created_at", + "value": cursor["created_at"], + }, + {"name": "@cursor_id", "value": cursor["id"]}, + ]) + query = ( + f"SELECT TOP @page_limit * FROM c WHERE {' AND '.join(clauses)} " + "ORDER BY c.created_at DESC, c.id DESC" + ) + return query, parameters + + +def _query_data_management_history_page( + list_kind, + page_size, + continuation_token=None, + filters=None, +): + default_page_size = DATA_MANAGEMENT_DEFAULT_JOB_LIMIT + safe_page_size = _safe_int( + page_size, + default=default_page_size, + minimum=1, + maximum=DATA_MANAGEMENT_HISTORY_MAX_PAGE_SIZE, + ) + normalized_filters = normalize_data_management_history_filters(list_kind, filters) + cursor = _decode_data_management_history_token( + continuation_token, + list_kind, + normalized_filters, + safe_page_size, + ) + query, parameters = _build_data_management_history_query( + list_kind, + normalized_filters, + safe_page_size + 1, + cursor=cursor, + ) + query_iterable = cosmos_data_management_jobs_container.query_items( query=query, parameters=parameters, enable_cross_partition_query=True, - max_item_count=safe_limit, - ))[:safe_limit] + max_item_count=safe_page_size + 1, + ) + results = list(query_iterable) + has_more = len(results) > safe_page_size + items = results[:safe_page_size] + next_cursor = None + if has_more and items: + last_item = items[-1] + next_cursor = { + "created_at": last_item.get("created_at"), + "id": last_item.get("id"), + } + next_token = _encode_data_management_history_token( + list_kind, + normalized_filters, + safe_page_size, + next_cursor, + ) + return { + "items": items, + "pagination": { + "page_size": safe_page_size, + "returned_count": len(items), + "has_more": bool(next_token), + "next_token": next_token, + }, + "filters": normalized_filters, + } + + +def get_data_management_jobs_page( + page_size=DATA_MANAGEMENT_DEFAULT_JOB_LIMIT, + continuation_token=None, + filters=None, +): + page = _query_data_management_history_page( + "jobs", + page_size, + continuation_token=continuation_token, + filters=filters, + ) + page["items"] = [ + public_job + for public_job in ( + sanitize_data_management_job_for_admin(job) + for job in page["items"] + ) + if public_job + ] + page["pagination"]["returned_count"] = len(page["items"]) + return page + + +def get_data_management_jobs(limit=DATA_MANAGEMENT_DEFAULT_JOB_LIMIT): + return get_data_management_jobs_page(page_size=limit)["items"] def get_recoverable_data_management_jobs( @@ -8863,49 +9172,120 @@ def sanitize_data_management_backup_for_admin(job): def get_data_management_backup_inventory(limit=100): - safe_limit = _safe_int(limit, default=100, minimum=1, maximum=500) - query = "SELECT * FROM c WHERE c.type = @type AND c.operation = @operation ORDER BY c.created_at DESC" - parameters = [ - {"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE}, - {"name": "@operation", "value": DATA_MANAGEMENT_OPERATION_BACKUP}, + page = _query_data_management_history_page("backups", limit) + return [ + backup + for backup in ( + sanitize_data_management_backup_for_admin(job) + for job in page["items"] + ) + if backup ] - jobs = list(cosmos_data_management_jobs_container.query_items( - query=query, - parameters=parameters, - enable_cross_partition_query=True, - max_item_count=safe_limit, - ))[:safe_limit] - return [backup for backup in (sanitize_data_management_backup_for_admin(job) for job in jobs) if backup] -def get_data_management_backup_summary(limit=100): - backups = get_data_management_backup_inventory(limit=limit) +def _get_data_management_backup_global_summary(): summary = { "full": 0, "partial": 0, "available": 0, "running": 0, "failed": 0, - "total": len(backups), + "total": 0, "latest_full": None, "latest_partial": None, } - for backup in backups: - status = backup.get("status") - backup_type = backup.get("backup_type") - if status in {DATA_MANAGEMENT_STATUS_COMPLETED, DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS}: - summary["available"] += 1 + aggregate_query = ( + "SELECT c.backup_type, c.status, COUNT(1) AS count FROM c " + "WHERE c.type = @type AND c.operation = @operation " + "GROUP BY c.backup_type, c.status" + ) + parameters = [ + {"name": "@type", "value": DATA_MANAGEMENT_JOB_TYPE}, + {"name": "@operation", "value": DATA_MANAGEMENT_OPERATION_BACKUP}, + ] + aggregate_rows = list(cosmos_data_management_jobs_container.query_items( + query=aggregate_query, + parameters=parameters, + enable_cross_partition_query=True, + )) + for row in aggregate_rows: + if not isinstance(row, dict): + continue + count = _safe_int(row.get("count"), default=0, minimum=0) + status = row.get("status") + backup_type = row.get("backup_type") + summary["total"] += count + if status in { + DATA_MANAGEMENT_STATUS_COMPLETED, + DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS, + }: + summary["available"] += count if backup_type == DATA_MANAGEMENT_BACKUP_FULL: - summary["full"] += 1 - summary["latest_full"] = summary["latest_full"] or backup + summary["full"] += count elif backup_type == DATA_MANAGEMENT_BACKUP_PARTIAL: - summary["partial"] += 1 - summary["latest_partial"] = summary["latest_partial"] or backup + summary["partial"] += count elif status == DATA_MANAGEMENT_STATUS_RUNNING: - summary["running"] += 1 + summary["running"] += count elif status == DATA_MANAGEMENT_STATUS_FAILED: - summary["failed"] += 1 - return {"summary": summary, "backups": backups} + summary["failed"] += count + + for backup_type, summary_field in ( + (DATA_MANAGEMENT_BACKUP_FULL, "latest_full"), + (DATA_MANAGEMENT_BACKUP_PARTIAL, "latest_partial"), + ): + latest_query = ( + "SELECT TOP 1 * FROM c WHERE c.type = @type " + "AND c.operation = @operation AND c.backup_type = @backup_type " + "AND (c.status = @completed OR c.status = @completed_with_warnings) " + "ORDER BY c.created_at DESC, c.id DESC" + ) + latest_parameters = parameters + [ + {"name": "@backup_type", "value": backup_type}, + {"name": "@completed", "value": DATA_MANAGEMENT_STATUS_COMPLETED}, + { + "name": "@completed_with_warnings", + "value": DATA_MANAGEMENT_STATUS_COMPLETED_WITH_WARNINGS, + }, + ] + latest_jobs = list(cosmos_data_management_jobs_container.query_items( + query=latest_query, + parameters=latest_parameters, + enable_cross_partition_query=True, + max_item_count=1, + )) + if latest_jobs: + summary[summary_field] = sanitize_data_management_backup_for_admin( + latest_jobs[0] + ) + return summary + + +def get_data_management_backup_summary( + limit=DATA_MANAGEMENT_DEFAULT_JOB_LIMIT, + continuation_token=None, + filters=None, +): + page = _query_data_management_history_page( + "backups", + limit, + continuation_token=continuation_token, + filters=filters, + ) + backups = [ + backup + for backup in ( + sanitize_data_management_backup_for_admin(job) + for job in page["items"] + ) + if backup + ] + page["pagination"]["returned_count"] = len(backups) + return { + "summary": _get_data_management_backup_global_summary(), + "backups": backups, + "pagination": page["pagination"], + "filters": page["filters"], + } def _search_text_matches(document, fields, search_text): diff --git a/application/single_app/route_backend_data_management.py b/application/single_app/route_backend_data_management.py index 5ee4437ad..691064cd3 100644 --- a/application/single_app/route_backend_data_management.py +++ b/application/single_app/route_backend_data_management.py @@ -14,6 +14,7 @@ DATA_MANAGEMENT_OPERATION_MIGRATION, DATA_MANAGEMENT_OPERATION_RESTORE, DataManagementCosmosEditorError, + DataManagementHistoryPaginationError, DataManagementSettingsValidationError, export_data_management_migration_manifest, generate_data_management_encryption_key, @@ -22,7 +23,7 @@ get_data_management_backup_summary, get_data_management_job_detail, get_data_management_job_progress, - get_data_management_jobs, + get_data_management_jobs_page, get_data_management_migration_catalog, get_data_management_settings, log_data_management_cosmos_editor_activity, @@ -47,6 +48,11 @@ from swagger_wrapper import get_auth_security, swagger_route +DATA_MANAGEMENT_HISTORY_VALIDATION_ERROR = ( + "Data Management history filters or continuation token are invalid." +) + + def _get_admin_context(): admin_user = session.get("user", {}) if session else {} admin_email = admin_user.get("preferred_username") or admin_user.get("email") or "unknown" @@ -76,6 +82,20 @@ def _get_cosmos_editor_payload(): return payload if isinstance(payload, dict) else {} +def _get_history_filters(list_kind): + filters = { + "status": request.args.get("status"), + "scheduled": request.args.get("scheduled"), + "created_from": request.args.get("created_from"), + "created_to": request.args.get("created_to"), + } + if list_kind == "jobs": + filters["operation"] = request.args.get("operation") + else: + filters["backup_type"] = request.args.get("backup_type") + return filters + + def _cosmos_editor_error_status(exc, default_status=400): status_code = getattr(exc, "status_code", None) if status_code == 404: @@ -403,9 +423,24 @@ def save_admin_data_management_cosmos_editor_document(): @login_required @admin_required def list_admin_data_management_jobs(): - limit = request.args.get("limit", 25) - jobs = [sanitize_data_management_job_for_admin(job) for job in get_data_management_jobs(limit=limit)] - return jsonify({"success": True, "jobs": jobs}), 200 + page_size = request.args.get("page_size", request.args.get("limit", 25)) + try: + page = get_data_management_jobs_page( + page_size=page_size, + continuation_token=request.args.get("continuation_token"), + filters=_get_history_filters("jobs"), + ) + except DataManagementHistoryPaginationError: + return jsonify({ + "success": False, + "error": DATA_MANAGEMENT_HISTORY_VALIDATION_ERROR, + }), 400 + return jsonify({ + "success": True, + "jobs": page["items"], + "pagination": page["pagination"], + "filters": page["filters"], + }), 200 @bp.route("/api/admin/data-management/jobs/", methods=["GET"]) @swagger_route(security=get_auth_security()) @@ -541,8 +576,18 @@ def cancel_admin_data_management_migration_job(job_id): @login_required @admin_required def list_admin_data_management_backups(): - limit = request.args.get("limit", 100) - backup_summary = get_data_management_backup_summary(limit=limit) + page_size = request.args.get("page_size", request.args.get("limit", 25)) + try: + backup_summary = get_data_management_backup_summary( + limit=page_size, + continuation_token=request.args.get("continuation_token"), + filters=_get_history_filters("backups"), + ) + except DataManagementHistoryPaginationError: + return jsonify({ + "success": False, + "error": DATA_MANAGEMENT_HISTORY_VALIDATION_ERROR, + }), 400 return jsonify({"success": True, **backup_summary}), 200 @bp.route("/api/admin/data-management/migration/catalog/", methods=["GET"]) diff --git a/application/single_app/static/js/admin/admin_data_management.js b/application/single_app/static/js/admin/admin_data_management.js index 0d45e4cff..277d1e253 100644 --- a/application/single_app/static/js/admin/admin_data_management.js +++ b/application/single_app/static/js/admin/admin_data_management.js @@ -11,7 +11,6 @@ const migrationMirrorConfirmationPhrase = "MIRROR WITH DELETIONS"; const elements = {}; let dataManagementModified = false; let storedBackupConnectionStringAvailable = false; -let backupInventory = []; let currentBackupFilter = "all"; let currentJobDetailId = null; let jobDetailRefreshTimer = null; @@ -27,6 +26,10 @@ let pendingDataManagementCancellationJob = null; const jobDetailRefreshIntervalMs = 2000; const activeJobStatuses = new Set(["queued", "running"]); +const historyListState = { + backups: createHistoryListState(), + jobs: createHistoryListState(), +}; const migrationTargetTypes = ["users", "groups", "public_workspaces"]; const migrationSelections = { users: new Map(), @@ -163,8 +166,25 @@ function bindElements() { "data-management-partial-backup-count", "data-management-available-backup-count", "data-management-backups-tbody", + "data_management_backup_status_filter", + "data_management_backup_scheduled_filter", + "data_management_backup_created_from", + "data_management_backup_created_to", + "data_management_backup_page_size", + "data-management-backup-pagination-status", + "data-management-backup-previous-page-btn", + "data-management-backup-next-page-btn", "data-management-refresh-jobs-btn", "data-management-jobs-tbody", + "data_management_job_operation_filter", + "data_management_job_status_filter", + "data_management_job_scheduled_filter", + "data_management_job_created_from", + "data_management_job_created_to", + "data_management_job_page_size", + "data-management-job-pagination-status", + "data-management-job-previous-page-btn", + "data-management-job-next-page-btn", "data-management-job-detail-modal", "data-management-job-detail-title", "data-management-job-detail-subtitle", @@ -250,10 +270,29 @@ function bindEvents() { elements.dataManagementRefreshMigrationSummaryBtn?.addEventListener("click", () => loadMigrationSummary(elements.dataManagementRefreshMigrationSummaryBtn, true)); bindMigrationPickerEvents(); elements.dataManagementRefreshBackupsBtn?.addEventListener("click", loadDataManagementBackups); - elements.dataManagementViewFullBackupsBtn?.addEventListener("click", () => setBackupFilter("full")); - elements.dataManagementViewPartialBackupsBtn?.addEventListener("click", () => setBackupFilter("partial")); - elements.dataManagementViewAllBackupsBtn?.addEventListener("click", () => setBackupFilter("all")); + elements.dataManagementViewFullBackupsBtn?.addEventListener("click", () => setBackupFilter("full", { load: true })); + elements.dataManagementViewPartialBackupsBtn?.addEventListener("click", () => setBackupFilter("partial", { load: true })); + elements.dataManagementViewAllBackupsBtn?.addEventListener("click", () => setBackupFilter("all", { load: true })); + [ + elements.datamanagementbackupstatusfilter, + elements.datamanagementbackupscheduledfilter, + elements.datamanagementbackupcreatedfrom, + elements.datamanagementbackupcreatedto, + elements.datamanagementbackuppagesize, + ].forEach((filter) => filter?.addEventListener("change", () => resetAndLoadHistory("backups"))); + elements.dataManagementBackupPreviousPageBtn?.addEventListener("click", () => loadPreviousHistoryPage("backups")); + elements.dataManagementBackupNextPageBtn?.addEventListener("click", () => loadNextHistoryPage("backups")); elements.dataManagementRefreshJobsBtn?.addEventListener("click", loadDataManagementJobs); + [ + elements.datamanagementjoboperationfilter, + elements.datamanagementjobstatusfilter, + elements.datamanagementjobscheduledfilter, + elements.datamanagementjobcreatedfrom, + elements.datamanagementjobcreatedto, + elements.datamanagementjobpagesize, + ].forEach((filter) => filter?.addEventListener("change", () => resetAndLoadHistory("jobs"))); + elements.dataManagementJobPreviousPageBtn?.addEventListener("click", () => loadPreviousHistoryPage("jobs")); + elements.dataManagementJobNextPageBtn?.addEventListener("click", () => loadNextHistoryPage("jobs")); elements.dataManagementJobDetailModal?.addEventListener("hidden.bs.modal", () => stopJobDetailAutoRefresh({ clearJob: true })); elements.dataManagementMigrationCancelModal?.addEventListener("hidden.bs.modal", () => { pendingDataManagementCancellationJob = null; @@ -1725,39 +1764,238 @@ function formatOperation(operation, backupType) { return operation.replace(/_/g, " "); } -async function loadDataManagementJobs() { - setBusy(elements.dataManagementRefreshJobsBtn, true, "Refreshing..."); - try { - const data = await requestJson("/api/admin/data-management/jobs?limit=25", { method: "GET" }); - renderJobs(data.jobs || []); - } catch (error) { - renderJobMessage(error.message || "Job history could not be loaded.", "danger"); - } finally { - setBusy(elements.dataManagementRefreshJobsBtn, false); +function createHistoryListState() { + return { + currentToken: null, + nextToken: null, + previousTokens: [], + pageNumber: 1, + requestGeneration: 0, + abortController: null, + }; +} + +function getHistoryListElements(listKind) { + if (listKind === "backups") { + return { + paginationStatus: elements.dataManagementBackupPaginationStatus, + previousButton: elements.dataManagementBackupPreviousPageBtn, + nextButton: elements.dataManagementBackupNextPageBtn, + refreshButton: elements.dataManagementRefreshBackupsBtn, + }; } + return { + paginationStatus: elements.dataManagementJobPaginationStatus, + previousButton: elements.dataManagementJobPreviousPageBtn, + nextButton: elements.dataManagementJobNextPageBtn, + refreshButton: elements.dataManagementRefreshJobsBtn, + }; } -async function loadDataManagementBackups() { - setBusy(elements.dataManagementRefreshBackupsBtn, true, "Refreshing..."); +function getHistoryListFilters(listKind) { + if (listKind === "backups") { + return { + backup_type: currentBackupFilter === "all" ? "" : currentBackupFilter, + status: elements.datamanagementbackupstatusfilter?.value || "", + scheduled: elements.datamanagementbackupscheduledfilter?.value || "all", + created_from: elements.datamanagementbackupcreatedfrom?.value || "", + created_to: elements.datamanagementbackupcreatedto?.value || "", + page_size: elements.datamanagementbackuppagesize?.value || "25", + }; + } + return { + operation: elements.datamanagementjoboperationfilter?.value || "", + status: elements.datamanagementjobstatusfilter?.value || "", + scheduled: elements.datamanagementjobscheduledfilter?.value || "all", + created_from: elements.datamanagementjobcreatedfrom?.value || "", + created_to: elements.datamanagementjobcreatedto?.value || "", + page_size: elements.datamanagementjobpagesize?.value || "25", + }; +} + +function buildHistoryListUrl(listKind, filters = getHistoryListFilters(listKind)) { + const state = historyListState[listKind]; + const params = new URLSearchParams(); + Object.entries(filters).forEach(([key, value]) => { + if (value !== "") { + params.set(key, value); + } + }); + if (state.currentToken) { + params.set("continuation_token", state.currentToken); + } + return `/api/admin/data-management/${listKind}?${params.toString()}`; +} + +function getHistoryDateRangeError(filters) { + const createdFrom = filters.created_from || ""; + const createdTo = filters.created_to || ""; + if (Boolean(createdFrom) !== Boolean(createdTo)) { + return "Select both Created from and Created through to apply a date range."; + } + if (!createdFrom) { + return ""; + } + const start = new Date(`${createdFrom}T00:00:00Z`); + const end = new Date(`${createdTo}T00:00:00Z`); + if (end < start) { + return "Created through must be on or after Created from."; + } + const rangeDays = Math.floor((end.getTime() - start.getTime()) / 86400000) + 1; + if (rangeDays > 366) { + return "Date range cannot exceed 366 days."; + } + return ""; +} + +function resetHistoryListState(listKind) { + const state = historyListState[listKind]; + state.abortController?.abort(); + state.currentToken = null; + state.nextToken = null; + state.previousTokens = []; + state.pageNumber = 1; +} + +function resetAndLoadHistory(listKind) { + resetHistoryListState(listKind); + if (listKind === "backups") { + loadDataManagementBackups(); + } else { + loadDataManagementJobs(); + } +} + +function loadNextHistoryPage(listKind) { + const state = historyListState[listKind]; + if (!state.nextToken) { + return; + } + state.previousTokens.push(state.currentToken); + state.currentToken = state.nextToken; + state.nextToken = null; + state.pageNumber += 1; + if (listKind === "backups") { + loadDataManagementBackups(); + } else { + loadDataManagementJobs(); + } +} + +function loadPreviousHistoryPage(listKind) { + const state = historyListState[listKind]; + if (state.previousTokens.length === 0) { + return; + } + state.currentToken = state.previousTokens.pop() || null; + state.nextToken = null; + state.pageNumber = Math.max(1, state.pageNumber - 1); + if (listKind === "backups") { + loadDataManagementBackups(); + } else { + loadDataManagementJobs(); + } +} + +function updateHistoryPaginationControls(listKind, pagination = {}, message = "") { + const state = historyListState[listKind]; + const listElements = getHistoryListElements(listKind); + const returnedCount = Number(pagination.returned_count || 0); + state.nextToken = pagination.next_token || null; + setButtonDisabled(listElements.previousButton, state.previousTokens.length === 0); + setButtonDisabled(listElements.nextButton, !pagination.has_more || !state.nextToken); + const suffix = pagination.has_more ? "" : " - final page"; + setText( + listElements.paginationStatus, + message || `Page ${state.pageNumber} - ${formatNumber(returnedCount)} record${returnedCount === 1 ? "" : "s"}${suffix}` + ); +} + +async function loadHistoryList(listKind) { + const state = historyListState[listKind]; + const listElements = getHistoryListElements(listKind); + const filters = getHistoryListFilters(listKind); + const dateRangeError = getHistoryDateRangeError(filters); + if (dateRangeError) { + state.abortController?.abort(); + state.requestGeneration += 1; + if (listKind === "backups") { + renderBackupMessage(dateRangeError, "danger"); + } else { + renderJobMessage(dateRangeError, "danger"); + } + updateHistoryPaginationControls(listKind, {}, dateRangeError); + return; + } + state.abortController?.abort(); + const controller = new AbortController(); + state.abortController = controller; + state.requestGeneration += 1; + const requestGeneration = state.requestGeneration; + setBusy(listElements.refreshButton, true, "Refreshing..."); + setText(listElements.paginationStatus, `Loading page ${state.pageNumber}...`); + setButtonDisabled(listElements.previousButton, true); + setButtonDisabled(listElements.nextButton, true); try { - const data = await requestJson("/api/admin/data-management/backups?limit=100", { method: "GET" }); - backupInventory = Array.isArray(data.backups) ? data.backups : []; - renderBackupSummary(data.summary || {}); - setBackupFilter(currentBackupFilter); + const data = await requestJson( + buildHistoryListUrl(listKind, filters), + { method: "GET", signal: controller.signal } + ); + if (requestGeneration !== state.requestGeneration) { + return; + } + const pagination = data.pagination || {}; + const items = listKind === "backups" ? data.backups || [] : data.jobs || []; + if ( + items.length === 0 + && !pagination.has_more + && state.pageNumber > 1 + && state.previousTokens.length > 0 + ) { + loadPreviousHistoryPage(listKind); + return; + } + if (listKind === "backups") { + renderBackups(items); + renderBackupSummary(data.summary || {}); + } else { + renderJobs(items); + } + updateHistoryPaginationControls(listKind, pagination); } catch (error) { - renderBackupMessage(error.message || "Backup inventory could not be loaded.", "danger"); + if (error.name === "AbortError" || requestGeneration !== state.requestGeneration) { + return; + } + const message = error.message || `${listKind === "backups" ? "Backup inventory" : "Job history"} could not be loaded.`; + if (listKind === "backups") { + renderBackupMessage(message, "danger"); + } else { + renderJobMessage(message, "danger"); + } + updateHistoryPaginationControls(listKind, {}, `Page ${state.pageNumber} could not be loaded.`); } finally { - setBusy(elements.dataManagementRefreshBackupsBtn, false); + if (requestGeneration === state.requestGeneration) { + setBusy(listElements.refreshButton, false); + state.abortController = null; + } } } +async function loadDataManagementJobs() { + await loadHistoryList("jobs"); +} + +async function loadDataManagementBackups() { + await loadHistoryList("backups"); +} + function renderBackupSummary(summary) { setText(elements.dataManagementFullBackupCount, formatNumber(summary.full || 0)); setText(elements.dataManagementPartialBackupCount, formatNumber(summary.partial || 0)); setText(elements.dataManagementAvailableBackupCount, formatNumber(summary.available || 0)); } -function setBackupFilter(filterValue) { +function setBackupFilter(filterValue, options = {}) { currentBackupFilter = ["all", "full", "partial"].includes(filterValue) ? filterValue : "all"; [ elements.dataManagementViewAllBackupsBtn, @@ -1773,25 +2011,23 @@ function setBackupFilter(filterValue) { element.classList.toggle("text-white-50", selected); }); }); - renderBackups(); + if (options.load) { + resetAndLoadHistory("backups"); + } } -function renderBackups() { +function renderBackups(backups) { const tbody = elements.dataManagementBackupsTbody; if (!tbody) { return; } - const availableBackups = backupInventory.filter(isAvailableBackup); - const filteredBackups = currentBackupFilter === "all" - ? availableBackups - : availableBackups.filter((backup) => backup.backup_type === currentBackupFilter); tbody.replaceChildren(); - if (filteredBackups.length === 0) { - renderBackupMessage("No completed backups match this view yet.", "muted"); + if (!Array.isArray(backups) || backups.length === 0) { + renderBackupMessage("No backups match the active filters.", "muted"); return; } - filteredBackups.forEach((backup) => { + backups.forEach((backup) => { tbody.appendChild(createBackupRow(backup)); }); } @@ -1892,10 +2128,6 @@ function createLabeledValue(label, value, allowBreak = false) { return wrapper; } -function isAvailableBackup(backup) { - return backup?.status === "completed" || backup?.status === "completed_with_warnings"; -} - function renderJobs(jobs) { const tbody = elements.dataManagementJobsTbody; if (!tbody) { diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index 7cbc707bf..8cfa1e9b8 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -6194,6 +6194,46 @@

Backup Inventory

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
@@ -6214,6 +6254,17 @@

Backup Inventory

+
@@ -6223,6 +6274,55 @@

Job History

Refresh
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
@@ -6242,6 +6342,17 @@

Job History

+