diff --git a/.impeccable/surfaces/application-single-app-templates-admin-settings-html.md b/.impeccable/surfaces/application-single-app-templates-admin-settings-html.md new file mode 100644 index 000000000..ca72d7f44 --- /dev/null +++ b/.impeccable/surfaces/application-single-app-templates-admin-settings-html.md @@ -0,0 +1,16 @@ +--- +version: 1 +slug: "application-single-app-templates-admin-settings-html" +primary_target: "application/single_app/templates/admin_settings.html" +related_targets: ["application/single_app/static/js/admin/admin_data_management.js","application/single_app/static/css/styles.css"] +--- + +Scope: Admin Settings Data Management migration workflow. Operate mode. + +Audience and job: A SimpleChat administrator configures a destination, selects large personal/group/public scopes, reviews authoritative impact and readiness, submits once, and monitors or recovers the durable migration. + +Primary task and proof: Move through Target, Scope, Content & Options, Review, Confirm, and Progress without losing valid selections. Server-owned counts, preflight checks, collision estimates, destructive state, and job telemetry provide proof. + +Direction: Extend the established Bootstrap admin system with a restrained control-room workflow. A persistent compact step rail makes risk and completion legible; one focused work panel replaces three simultaneous pickers. The memorable moment is the review checkpoint changing from incomplete to an evidence-backed ready or blocked state. + +Constraints: Preserve backup, Cosmos editor, existing settings storage, #1043 durable migration behavior, local browser assets, safe DOM construction, light/dark themes, keyboard access, and responsive behavior. Do not invent backend state in JavaScript or expose raw settings, credentials, endpoints, SAS values, or unsafe HTML. diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 000000000..72678a542 --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,61 @@ +# Product + + + +## Platform + +web + +## Users + +SimpleChat serves authenticated enterprise users who chat with generative AI grounded in personal, group, and public-workspace data. Administrators configure Azure-backed services, govern capabilities, and operate backup and migration workflows for production environments. + +## Product Purpose + +SimpleChat provides secure, context-aware generative AI using Azure OpenAI and retrieval-augmented generation. It lets organizations manage documents and AI-assisted work across personal, shared-group, and public workspaces while retaining administrative control over models, data, security, and operations. + +## Positioning + +SimpleChat combines Azure-native identity, Cosmos DB, Azure AI Search, Azure OpenAI, and document-processing services in one deployable application with explicit administrative governance and data-management workflows. + +## Operating Context + +- Users work with documents, conversations, agents, actions, and governed workspaces. +- Administrators operate the application through authenticated settings and control-center surfaces. +- Data migrations move selected SimpleChat users, groups, public workspaces, documents, Search records, and Enhanced Citation source blobs between SimpleChat environments. +- Migration work may be long-running, destructive, resumable, and operationally sensitive, so administrators need authoritative review, progress, and recovery information. + +## Capabilities and Constraints + +- Personal, group, and public-workspace document management and retrieval-augmented chat. +- Azure Cosmos DB, Azure AI Search, Azure OpenAI, Azure Storage, and Microsoft Entra integration. +- Admin-only backup, migration, settings, and governance operations. +- Durable migration checkpoints, retries, cancellation, provenance, reconciliation, and optional temporary Cosmos capacity management. +- Browser runtime JavaScript and companion assets are served locally; frontend settings and migration responses must not expose credentials or internal configuration. +- The web interface uses Bootstrap 5, supports light and dark themes, and must remain keyboard accessible and responsive. + +## Brand Commitments + +- Product name: SimpleChat. +- Preserve the existing SimpleChat logos and the familiar Bootstrap-based application vocabulary. +- Administrative copy is direct, operational, and explicit about risk; it does not use marketing language for destructive or production-impacting actions. + +## Evidence on Hand + +- Product overview and deployment facts: `README.md`. +- Existing application and admin interface: `application/single_app/`. +- Migration behavior and operational guidance: `docs/explanation/features/DATA_MANAGEMENT_BACKUP_MIGRATION.md` and `docs/explanation/features/DATA_MANAGEMENT_MIGRATION_RESILIENCE.md`. +- Product logos: `application/single_app/static/images/`. +- No customer claims, testimonials, pricing, or benchmark assets are approved for invention. + +## Product Principles + +- Keep data boundaries and authorization explicit. +- Make high-impact operations reviewable before execution. +- Prefer server-authoritative state and counts over browser inference. +- Preserve durable recovery paths for long-running work. +- Surface operational risk plainly without obscuring the task. + +## Accessibility & Inclusion + +Admin workflows must preserve semantic structure, visible labels, keyboard navigation, focus feedback, responsive layouts, and accessible loading, validation, progress, and status states. diff --git a/application/single_app/config.py b/application/single_app/config.py index 1d6b6bacf..9f8af2cc3 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -95,7 +95,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.104" +VERSION = "0.250.105" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_data_management.py b/application/single_app/functions_data_management.py index ebb9fc51c..e46f16504 100644 --- a/application/single_app/functions_data_management.py +++ b/application/single_app/functions_data_management.py @@ -1,8 +1,9 @@ # functions_data_management.py """Data Management settings, schedules, and durable job records.""" -import copy import base64 +import binascii +import copy from email.utils import parsedate_to_datetime import hashlib import hmac @@ -86,6 +87,8 @@ from functions_data_management_search_write_fence import ( acquire_data_management_search_write_fence, acquire_data_management_target_migration_coordinator, + inspect_data_management_search_write_gate, + inspect_data_management_target_migration_coordinator, release_data_management_search_write_fence, release_data_management_target_migration_coordinator, renew_data_management_search_write_fence, @@ -117,6 +120,8 @@ DATA_MANAGEMENT_JOB_TYPE = "data_management_job" DATA_MANAGEMENT_JOB_ITEM_TYPE = "data_management_job_item" DATA_MANAGEMENT_MIGRATION_MANIFEST_BATCH_TYPE = "data_management_migration_manifest_batch" +DATA_MANAGEMENT_MIGRATION_REVIEW_TYPE = "data_management_migration_review" +DATA_MANAGEMENT_MIGRATION_REVIEW_TTL_SECONDS = 900 DATA_MANAGEMENT_MIRROR_DELETION_BATCH_TYPE = "data_management_mirror_deletion_batch" DATA_MANAGEMENT_MIGRATION_LOCK_TYPE = "data_management_migration_lock" DATA_MANAGEMENT_BACKUP_MANIFEST_BATCH_TYPE = "data_management_backup_manifest_batch" @@ -176,7 +181,10 @@ DATA_MANAGEMENT_MIGRATION_LOCK_PREFIX = "data_management_migration_lock" DATA_MANAGEMENT_BACKUP_LOCK_PREFIX = "data_management_backup_lock" DATA_MANAGEMENT_BACKUP_SOURCE_SCOPE = "simplechat-primary" -DATA_MANAGEMENT_MIGRATION_CATALOG_LIMIT = 50 +DATA_MANAGEMENT_MIGRATION_CATALOG_LIMIT = 25 +DATA_MANAGEMENT_MIGRATION_CATALOG_MAX_LIMIT = 100 +DATA_MANAGEMENT_MIGRATION_CATALOG_CURSOR_VERSION = 1 +DATA_MANAGEMENT_MIGRATION_MAX_SELECTED_IDS = 2000 DATA_MANAGEMENT_MIGRATION_BATCH_SIZE = 500 DATA_MANAGEMENT_MIGRATION_MANIFEST_BATCH_SIZE = 100 DATA_MANAGEMENT_BACKUP_MANIFEST_BATCH_SIZE = 100 @@ -1082,7 +1090,16 @@ def _normalize_migration_selection(selection=None): mode = _safe_text(selection.get("mode"), "none") if mode not in {"none", "all", "selected"}: mode = "none" - ids = _safe_list(selection.get("ids"), limit=2000) + raw_ids = selection.get("ids") + if ( + mode == "selected" and + isinstance(raw_ids, list) and + len(raw_ids) > DATA_MANAGEMENT_MIGRATION_MAX_SELECTED_IDS + ): + raise DataManagementSettingsValidationError( + f"Selected migration scopes cannot exceed {DATA_MANAGEMENT_MIGRATION_MAX_SELECTED_IDS} IDs." + ) + ids = _safe_list(raw_ids, limit=DATA_MANAGEMENT_MIGRATION_MAX_SELECTED_IDS) if mode == "selected" and not ids: mode = "none" return { @@ -1940,9 +1957,13 @@ def _update_migration_state_totals(state): "collision_count", "processed_count", "bytes", - "artifact_bytes", ): totals[field_name] += _safe_int(metrics.get(field_name), default=0, minimum=0) + totals["bytes"] += _safe_int( + metrics.get("artifact_bytes"), + default=0, + minimum=0, + ) try: totals["request_units"] += max(0.0, float(metrics.get("request_units") or 0.0)) except (TypeError, ValueError): @@ -9288,22 +9309,158 @@ def get_data_management_backup_summary( } -def _search_text_matches(document, fields, search_text): - normalized_search = _safe_text(search_text).lower() +def _get_migration_catalog_definition(target_type): + normalized_target_type = _safe_text(target_type) + definitions = { + "users": { + "container": getattr(app_config, "cosmos_user_settings_container", None), + "document_container": getattr(app_config, "cosmos_user_documents_container", None), + "document_scope_field": "user_id", + "search_fields": ("email", "display_name", "id"), + "label_fields": ("display_name", "email", "id"), + "description_fields": ("email",), + "empty_description": "No email recorded", + }, + "groups": { + "container": getattr(app_config, "cosmos_groups_container", None), + "document_container": getattr(app_config, "cosmos_group_documents_container", None), + "document_scope_field": "group_id", + "search_fields": ("name", "description", "id"), + "label_fields": ("name", "id"), + "description_fields": ("description",), + "empty_description": "No description recorded", + }, + "public_workspaces": { + "container": getattr(app_config, "cosmos_public_workspaces_container", None), + "document_container": getattr(app_config, "cosmos_public_documents_container", None), + "document_scope_field": "public_workspace_id", + "search_fields": ("name", "description", "id"), + "label_fields": ("name", "id"), + "description_fields": ("description",), + "empty_description": "No description recorded", + }, + } + definition = definitions.get(normalized_target_type) + if not definition: + raise DataManagementSettingsValidationError("Unsupported migration catalog type.") + if definition.get("container") is None: + raise DataManagementSettingsValidationError( + f"Migration catalog '{normalized_target_type}' is not initialized." + ) + return normalized_target_type, definition + + +def _build_migration_catalog_search_clause(search_fields, search_text): + normalized_search = _safe_text(search_text).lower()[:200] if not normalized_search: - return True - return any(normalized_search in _safe_text(document.get(field_name)).lower() for field_name in fields) + return "", [] + clauses = [ + f"CONTAINS(c.{field_name}, @catalog_search, true)" + for field_name in search_fields + ] + return f"({' OR '.join(clauses)})", [ + {"name": "@catalog_search", "value": normalized_search}, + ] -def _query_catalog_items(container, search_text, search_fields, order_field, limit=DATA_MANAGEMENT_MIGRATION_CATALOG_LIMIT): - safe_limit = _safe_int(limit, default=DATA_MANAGEMENT_MIGRATION_CATALOG_LIMIT, minimum=1, maximum=250) - results = [] +def _encode_migration_catalog_cursor(target_type, search_text, last_id): + payload = { + "v": DATA_MANAGEMENT_MIGRATION_CATALOG_CURSOR_VERSION, + "type": target_type, + "search": _safe_text(search_text).lower()[:200], + "last_id": _safe_text(last_id), + } + encoded = base64.urlsafe_b64encode(json.dumps( + payload, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8")).decode("ascii") + return encoded.rstrip("=") + + +def _decode_migration_catalog_cursor(cursor, target_type, search_text): + normalized_cursor = _safe_text(cursor) + if not normalized_cursor: + return "" + if len(normalized_cursor) > 4096: + raise DataManagementSettingsValidationError("Migration catalog continuation is invalid.") + try: + padding = "=" * (-len(normalized_cursor) % 4) + payload = json.loads(base64.urlsafe_b64decode( + f"{normalized_cursor}{padding}".encode("ascii") + ).decode("utf-8")) + except ( + binascii.Error, + json.JSONDecodeError, + TypeError, + UnicodeDecodeError, + ValueError, + ) as exc: + raise DataManagementSettingsValidationError( + "Migration catalog continuation is invalid." + ) from exc + expected_search = _safe_text(search_text).lower()[:200] + if ( + not isinstance(payload, dict) or + payload.get("v") != DATA_MANAGEMENT_MIGRATION_CATALOG_CURSOR_VERSION or + payload.get("type") != target_type or + payload.get("search") != expected_search or + not _safe_text(payload.get("last_id")) + ): + raise DataManagementSettingsValidationError( + "Migration catalog continuation does not match the current search." + ) + return _safe_text(payload.get("last_id")) + + +def _query_migration_catalog_count(container, search_fields, search_text): + search_clause, parameters = _build_migration_catalog_search_clause( + search_fields, + search_text, + ) + query = "SELECT VALUE COUNT(1) FROM c" + if search_clause: + query = f"{query} WHERE {search_clause}" + results = list(container.query_items( + query=query, + parameters=parameters, + enable_cross_partition_query=True, + )) + return results[0] if results and isinstance(results[0], int) else 0 + + +def _query_migration_catalog_page( + container, + search_fields, + search_text, + after_id, + page_size, +): + search_clause, parameters = _build_migration_catalog_search_clause( + search_fields, + search_text, + ) + conditions = [] + if search_clause: + conditions.append(search_clause) + if after_id: + conditions.append("c.id > @catalog_after_id") + parameters.append({"name": "@catalog_after_id", "value": after_id}) query = "SELECT * FROM c" - for item in container.query_items(query=query, enable_cross_partition_query=True): - if _search_text_matches(item, search_fields, search_text): - results.append(_strip_cosmos_system_fields(item)) - results.sort(key=lambda item: _safe_text(item.get(order_field) or item.get("name") or item.get("display_name") or item.get("email") or item.get("id")).lower()) - return results[:safe_limit] + if conditions: + query = f"{query} WHERE {' AND '.join(conditions)}" + query = f"{query} ORDER BY c.id" + results = [] + for item in container.query_items( + query=query, + parameters=parameters, + enable_cross_partition_query=True, + ): + results.append(_strip_cosmos_system_fields(item)) + if len(results) > page_size: + break + return results def _count_documents_for_scope(container, field_name, scope_id): @@ -9313,72 +9470,87 @@ def _count_documents_for_scope(container, field_name, scope_id): return results[0] if results and isinstance(results[0], int) else 0 -def get_data_management_migration_catalog(target_type, search_text="", limit=DATA_MANAGEMENT_MIGRATION_CATALOG_LIMIT): - normalized_target_type = _safe_text(target_type) - if normalized_target_type == "users": - users = _query_catalog_items( - app_config.cosmos_user_settings_container, - search_text, - ["email", "display_name", "id"], - "display_name", - limit=limit, +def get_data_management_migration_catalog( + target_type, + search_text="", + limit=DATA_MANAGEMENT_MIGRATION_CATALOG_LIMIT, + continuation_token="", +): + normalized_target_type, definition = _get_migration_catalog_definition( + target_type + ) + normalized_search = _safe_text(search_text).lower()[:200] + safe_limit = _safe_int( + limit, + default=DATA_MANAGEMENT_MIGRATION_CATALOG_LIMIT, + minimum=1, + maximum=DATA_MANAGEMENT_MIGRATION_CATALOG_MAX_LIMIT, + ) + after_id = _decode_migration_catalog_cursor( + continuation_token, + normalized_target_type, + normalized_search, + ) + total_count = _query_migration_catalog_count( + definition["container"], + definition["search_fields"], + normalized_search, + ) + page = _query_migration_catalog_page( + definition["container"], + definition["search_fields"], + normalized_search, + after_id, + safe_limit, + ) + has_more = len(page) > safe_limit + page = page[:safe_limit] + items = [] + for item in page: + item_id = _safe_text(item.get("id")) + if not item_id: + continue + label = next( + ( + _safe_text(item.get(field_name)) + for field_name in definition["label_fields"] + if _safe_text(item.get(field_name)) + ), + item_id, ) - return { - "type": "users", - "items": [ - { - "id": user.get("id"), - "label": user.get("display_name") or user.get("email") or user.get("id"), - "description": user.get("email") or "No email recorded", - "document_count": _count_documents_for_scope(app_config.cosmos_user_documents_container, "user_id", user.get("id")), - } - for user in users - if user.get("id") - ], - } - if normalized_target_type == "groups": - groups = _query_catalog_items( - app_config.cosmos_groups_container, - search_text, - ["name", "description", "id"], - "name", - limit=limit, + description = next( + ( + _safe_text(item.get(field_name)) + for field_name in definition["description_fields"] + if _safe_text(item.get(field_name)) + ), + definition["empty_description"], ) - return { - "type": "groups", - "items": [ - { - "id": group.get("id"), - "label": group.get("name") or group.get("id"), - "description": group.get("description") or "No description recorded", - "document_count": _count_documents_for_scope(app_config.cosmos_group_documents_container, "group_id", group.get("id")), - } - for group in groups - if group.get("id") - ], - } - if normalized_target_type == "public_workspaces": - workspaces = _query_catalog_items( - app_config.cosmos_public_workspaces_container, - search_text, - ["name", "description", "id"], - "name", - limit=limit, + items.append({ + "id": item_id, + "label": label, + "description": description, + "document_count": _count_documents_for_scope( + definition["document_container"], + definition["document_scope_field"], + item_id, + ), + }) + next_continuation_token = "" + if has_more and page: + next_continuation_token = _encode_migration_catalog_cursor( + normalized_target_type, + normalized_search, + page[-1].get("id"), ) - return { - "type": "public_workspaces", - "items": [ - { - "id": workspace.get("id"), - "label": workspace.get("name") or workspace.get("id"), - "description": workspace.get("description") or "No description recorded", - "document_count": _count_documents_for_scope(app_config.cosmos_public_documents_container, "public_workspace_id", workspace.get("id")), - } - for workspace in workspaces - if workspace.get("id") - ], - } - raise DataManagementSettingsValidationError("Unsupported migration catalog type.") + return { + "type": normalized_target_type, + "items": items, + "total_count": total_count, + "page_size": safe_limit, + "has_more": has_more, + "continuation_token": next_continuation_token, + } def _dedupe_limited_strings(values, limit=500): @@ -9397,7 +9569,10 @@ def _dedupe_limited_strings(values, limit=500): return ordered_values -def normalize_data_management_migration_plan(options): +def normalize_data_management_migration_plan( + options, + require_mirror_confirmation=True, +): raw_plan = options.get("migration_plan") if isinstance(options, dict) else {} if not isinstance(raw_plan, dict): raw_plan = {} @@ -9424,6 +9599,7 @@ def normalize_data_management_migration_plan(options): ) if ( migration_mode == DATA_MANAGEMENT_MIGRATION_MODE_MIRROR and + require_mirror_confirmation and not mirror_deletions_confirmed ): raise DataManagementSettingsValidationError( @@ -9449,22 +9625,84 @@ def normalize_data_management_migration_plan(options): def _resolve_plan_scope_ids(target_type, plan_entry): + if plan_entry.get("mode") != "selected": + return [] + return _dedupe_limited_strings( + plan_entry.get("ids"), + limit=DATA_MANAGEMENT_MIGRATION_MAX_SELECTED_IDS, + ) + + +def _count_migration_scope_documents(target_type, plan_entry): + if not plan_entry.get("include_documents"): + return 0 + document_definitions = { + "users": ( + getattr(app_config, "cosmos_user_documents_container", None), + "user_id", + ), + "groups": ( + getattr(app_config, "cosmos_group_documents_container", None), + "group_id", + ), + "public_workspaces": ( + getattr(app_config, "cosmos_public_documents_container", None), + "public_workspace_id", + ), + } + document_container, document_scope_field = document_definitions.get( + target_type, + (None, ""), + ) + if document_container is None: + return 0 if plan_entry.get("mode") == "all": - catalog = get_data_management_migration_catalog(target_type, limit=1000) - return [item.get("id") for item in catalog.get("items", []) if item.get("id")] - return _dedupe_limited_strings(plan_entry.get("ids")) + results = list(document_container.query_items( + query="SELECT VALUE COUNT(1) FROM c", + enable_cross_partition_query=True, + )) + return results[0] if results and isinstance(results[0], int) else 0 + return sum( + _count_documents_for_scope( + document_container, + document_scope_field, + scope_id, + ) + for scope_id in _resolve_plan_scope_ids(target_type, plan_entry) + ) -def summarize_data_management_migration_plan(options): - plan = normalize_data_management_migration_plan(options or {}) +def summarize_data_management_migration_plan( + options, + require_mirror_confirmation=True, +): + plan = normalize_data_management_migration_plan( + options or {}, + require_mirror_confirmation=require_mirror_confirmation, + ) summary = {} for target_type in DATA_MANAGEMENT_MIGRATION_TARGET_TYPES: - ids = _resolve_plan_scope_ids(target_type, plan[target_type]) + plan_entry = plan[target_type] + ids = _resolve_plan_scope_ids(target_type, plan_entry) + if plan_entry.get("mode") == "all": + _, definition = _get_migration_catalog_definition(target_type) + count = _query_migration_catalog_count( + definition["container"], + definition["search_fields"], + "", + ) + else: + count = len(ids) summary[target_type] = { - "mode": plan[target_type].get("mode"), - "count": len(ids), - "include_documents": bool(plan[target_type].get("include_documents")), + "mode": plan_entry.get("mode"), + "count": count, + "document_count": _count_migration_scope_documents( + target_type, + plan_entry, + ), + "include_documents": bool(plan_entry.get("include_documents")), "ids": ids[:50], + "ids_truncated": len(ids) > 50, } summary["include_ai_search"] = bool(plan.get("include_ai_search")) summary["include_source_blobs"] = bool(plan.get("include_source_blobs")) @@ -9477,6 +9715,605 @@ def summarize_data_management_migration_plan(options): return summary +def _migration_review_fingerprint(settings, migration_plan): + settings_fields = ( + "target_cosmos_authentication_type", + "target_cosmos_endpoint", + "target_cosmos_database_name", + "target_cosmos_key", + "target_cosmos_subscription_id", + "target_cosmos_resource_group", + "target_ai_search_authentication_type", + "target_ai_search_endpoint", + "target_ai_search_key", + "target_enhanced_citations_storage_authentication_type", + "target_enhanced_citations_storage_blob_endpoint", + "target_enhanced_citations_storage_connection_string", + "migration_max_parallel_operations", + "migration_retry_count", + "migration_skip_recent_within_hours", + "migration_temporary_destination_ru_enabled", + "migration_temporary_destination_ru", + ) + fingerprint_plan = copy.deepcopy(migration_plan) + fingerprint_plan.pop("mirror_deletions_confirmed", None) + fingerprint_payload = { + "migration_plan": fingerprint_plan, + "settings": { + field_name: settings.get(field_name) + for field_name in settings_fields + }, + } + return hashlib.sha256(json.dumps( + fingerprint_payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8")).hexdigest() + + +def get_data_management_migration_review_fingerprint( + settings=None, + migration_plan=None, +): + """Return the execution fingerprint for current settings and plan.""" + normalized_settings = _normalize_data_management_settings_from_payload( + settings + ) + normalized_plan = normalize_data_management_migration_plan( + { + "migration_plan": ( + migration_plan + if isinstance(migration_plan, dict) + else {} + ), + }, + require_mirror_confirmation=False, + ) + return _migration_review_fingerprint( + normalized_settings, + normalized_plan, + ) + + +def _migration_review_admin_hash(admin_user_id): + normalized_user_id = _safe_text(admin_user_id) + if not normalized_user_id or normalized_user_id == "unknown": + raise DataManagementSettingsValidationError( + "An authenticated administrator is required for migration review." + ) + return hashlib.sha256(normalized_user_id.encode("utf-8")).hexdigest() + + +def create_data_management_migration_review_authorization( + admin_user_id, + review_fingerprint, +): + """Persist one short-lived, admin-bound authorization for a ready review.""" + normalized_fingerprint = _safe_text(review_fingerprint) + if not normalized_fingerprint: + raise DataManagementSettingsValidationError( + "Migration review fingerprint is required." + ) + now = _now_utc() + authorization_id = str(uuid.uuid4()) + authorization = { + "id": authorization_id, + "type": DATA_MANAGEMENT_MIGRATION_REVIEW_TYPE, + "status": "ready", + "admin_hash": _migration_review_admin_hash(admin_user_id), + "review_fingerprint": normalized_fingerprint, + "created_at": now.isoformat(), + "expires_at": ( + now + timedelta( + seconds=DATA_MANAGEMENT_MIGRATION_REVIEW_TTL_SECONDS + ) + ).isoformat(), + } + created = cosmos_data_management_jobs_container.create_item( + body=authorization + ) + return { + "authorization_token": created.get("id", authorization_id), + "authorization_expires_at": created.get( + "expires_at", + authorization["expires_at"], + ), + } + + +def _read_data_management_migration_review_authorization( + authorization_token, + admin_user_id, + review_fingerprint, + allow_expired=False, +): + normalized_token = _safe_text(authorization_token) + normalized_fingerprint = _safe_text(review_fingerprint) + if not normalized_token or not normalized_fingerprint: + raise DataManagementSettingsValidationError( + "A current migration review authorization is required." + ) + try: + authorization = cosmos_data_management_jobs_container.read_item( + item=normalized_token, + partition_key=normalized_token, + ) + except ( + CosmosResourceNotFoundError, + KeyError, + ResourceNotFoundError, + ) as exc: + raise DataManagementSettingsValidationError( + "Migration review authorization is invalid or expired." + ) from exc + expires_at = _parse_iso_datetime(authorization.get("expires_at")) + if ( + authorization.get("type") != DATA_MANAGEMENT_MIGRATION_REVIEW_TYPE or + ( + not allow_expired and + (not expires_at or expires_at <= _now_utc()) + ) or + not hmac.compare_digest( + _safe_text(authorization.get("admin_hash")), + _migration_review_admin_hash(admin_user_id), + ) or + not hmac.compare_digest( + _safe_text(authorization.get("review_fingerprint")), + normalized_fingerprint, + ) + ): + raise DataManagementSettingsValidationError( + "Migration review authorization is invalid or expired." + ) + return authorization + + +def reserve_data_management_migration_review_authorization( + authorization_token, + admin_user_id, + review_fingerprint, +): + """Reserve one ready authorization before durable job creation.""" + authorization = _read_data_management_migration_review_authorization( + authorization_token, + admin_user_id, + review_fingerprint, + ) + if authorization.get("status") != "ready": + raise DataManagementSettingsValidationError( + "Migration review authorization is already reserved or used." + ) + reservation_token = uuid.uuid4().hex + job_id = str(uuid.uuid5( + uuid.NAMESPACE_URL, + f"simplechat-migration-review:{authorization['id']}", + )) + replacement = copy.deepcopy(authorization) + replacement.update({ + "status": "reserved", + "reservation_token": reservation_token, + "reserved_at": _now_iso(), + "job_id": job_id, + }) + try: + cosmos_data_management_jobs_container.replace_item( + item=authorization["id"], + body=replacement, + etag=authorization.get("_etag"), + match_condition=MatchConditions.IfNotModified, + ) + except Exception as exc: + if getattr(exc, "status_code", None) in {409, 412}: + raise DataManagementSettingsValidationError( + "Migration review authorization was already reserved or used." + ) from exc + raise + return { + "authorization_token": authorization["id"], + "reservation_token": reservation_token, + "job_id": job_id, + } + + +def release_data_management_migration_review_reservation( + authorization_token, + reservation_token, +): + """Release an exact reservation when durable job creation fails.""" + normalized_token = _safe_text(authorization_token) + normalized_reservation = _safe_text(reservation_token) + if not normalized_token or not normalized_reservation: + return False + try: + authorization = cosmos_data_management_jobs_container.read_item( + item=normalized_token, + partition_key=normalized_token, + ) + except ( + CosmosResourceNotFoundError, + KeyError, + ResourceNotFoundError, + ): + return False + if ( + authorization.get("status") != "reserved" or + not hmac.compare_digest( + _safe_text(authorization.get("reservation_token")), + normalized_reservation, + ) + ): + return False + replacement = copy.deepcopy(authorization) + replacement.update({ + "status": "ready", + "reservation_released_at": _now_iso(), + }) + replacement.pop("reservation_token", None) + replacement.pop("reserved_at", None) + replacement.pop("job_id", None) + try: + cosmos_data_management_jobs_container.replace_item( + item=normalized_token, + body=replacement, + etag=authorization.get("_etag"), + match_condition=MatchConditions.IfNotModified, + ) + except Exception as exc: + if getattr(exc, "status_code", None) in {409, 412}: + return False + log_event( + "[DataManagement] Failed to release migration review reservation.", + { + "authorization_id": normalized_token, + "error_type": type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return False + return True + + +def consume_data_management_migration_review_authorization( + authorization_token, + admin_user_id, + review_fingerprint, + reservation_token, + job_id, +): + """Consume an exact reservation before the reviewed job starts work.""" + authorization = _read_data_management_migration_review_authorization( + authorization_token, + admin_user_id, + review_fingerprint, + allow_expired=True, + ) + normalized_reservation = _safe_text(reservation_token) + normalized_job_id = _safe_text(job_id) + if ( + authorization.get("status") == "consumed" and + hmac.compare_digest( + _safe_text(authorization.get("consumed_job_id")), + normalized_job_id, + ) + ): + return True + if ( + authorization.get("status") != "reserved" or + not normalized_reservation or + not normalized_job_id or + not hmac.compare_digest( + _safe_text(authorization.get("reservation_token")), + normalized_reservation, + ) or + not hmac.compare_digest( + _safe_text(authorization.get("job_id")), + normalized_job_id, + ) + ): + raise DataManagementSettingsValidationError( + "Migration review reservation is invalid or already used." + ) + replacement = copy.deepcopy(authorization) + replacement.update({ + "status": "consumed", + "consumed_at": _now_iso(), + "consumed_job_id": normalized_job_id, + }) + replacement.pop("reservation_token", None) + try: + cosmos_data_management_jobs_container.replace_item( + item=authorization["id"], + body=replacement, + etag=authorization.get("_etag"), + match_condition=MatchConditions.IfNotModified, + ) + except Exception as exc: + if getattr(exc, "status_code", None) in {409, 412}: + raise DataManagementSettingsValidationError( + "Migration review reservation changed before execution." + ) from exc + raise + return True + + +def _migration_review_check( + check_id, + label, + workflow_step, + callback, + success_summary, +): + try: + details = callback() + except DataManagementSettingsValidationError as exc: + return { + "id": check_id, + "label": label, + "workflow_step": workflow_step, + "status": "block", + "summary": str(exc), + "details": None, + } + except Exception as exc: + log_event( + "[DataManagement] Migration review check failed.", + { + "check_id": check_id, + "error_type": type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return { + "id": check_id, + "label": label, + "workflow_step": workflow_step, + "status": "block", + "summary": f"{label} could not be verified.", + "details": None, + } + return { + "id": check_id, + "label": label, + "workflow_step": workflow_step, + "status": "pass", + "summary": success_summary, + "details": details, + } + + +def review_data_management_migration(settings=None, migration_plan=None): + """Build a sanitized, server-owned pre-execution migration review.""" + normalized_settings = _normalize_data_management_settings_from_payload(settings) + normalized_plan = normalize_data_management_migration_plan( + { + "migration_plan": ( + migration_plan + if isinstance(migration_plan, dict) + else {} + ), + }, + require_mirror_confirmation=False, + ) + summary = summarize_data_management_migration_plan({ + "migration_plan": normalized_plan, + }, require_mirror_confirmation=False) + selected_total = sum( + summary[target_type]["count"] + for target_type in DATA_MANAGEMENT_MIGRATION_TARGET_TYPES + ) + if selected_total == 0: + raise DataManagementSettingsValidationError( + "Choose at least one user, group, or public workspace before review." + ) + + checks = [] + checks.append({ + "id": "scope", + "label": "Migration scope", + "workflow_step": "scope", + "status": "pass", + "summary": f"{selected_total} principal scopes are included.", + "details": { + target_type: { + "mode": summary[target_type]["mode"], + "count": summary[target_type]["count"], + "document_count": summary[target_type]["document_count"], + "include_documents": summary[target_type]["include_documents"], + } + for target_type in DATA_MANAGEMENT_MIGRATION_TARGET_TYPES + }, + }) + checks.append(_migration_review_check( + "cosmos_access", + "Cosmos source and destination access", + "target", + lambda: _preflight_target_cosmos_migration_access( + normalized_settings, + normalized_plan, + ), + "Source reads, destination writes, and partition keys are ready.", + )) + + def inspect_destination_coordination(): + target_database = _get_target_cosmos_database(normalized_settings) + gate_container = _get_target_data_management_search_write_gate_container( + target_database + ) + return { + "migration_coordinator": ( + inspect_data_management_target_migration_coordinator( + gate_container + ) + ), + "search_write_gate": inspect_data_management_search_write_gate( + gate_container + ), + } + + coordination_check = _migration_review_check( + "destination_coordination", + "Destination migration coordination", + "review", + inspect_destination_coordination, + "No active destination migration lock blocks this plan.", + ) + coordination_details = coordination_check.get("details") or {} + if ( + coordination_check["status"] == "pass" and + ( + not (coordination_details.get("migration_coordinator") or {}).get( + "available", + True, + ) or + not (coordination_details.get("search_write_gate") or {}).get( + "available", + True, + ) + ) + ): + coordination_check.update({ + "status": "block", + "summary": "Another migration currently owns the destination coordination boundary.", + }) + checks.append(coordination_check) + + if normalized_plan.get("include_ai_search"): + checks.append(_migration_review_check( + "ai_search_access", + "AI Search readiness", + "target", + lambda: _preflight_target_ai_search_migration_access( + normalized_settings, + normalized_plan, + ), + "Selected source indexes and destination Search writes are ready.", + )) + else: + checks.append({ + "id": "ai_search_access", + "label": "AI Search readiness", + "workflow_step": "options", + "status": "warning", + "summary": "AI Search documents are excluded from this migration.", + "details": None, + }) + + if normalized_plan.get("include_source_blobs"): + checks.append(_migration_review_check( + "source_blob_access", + "Enhanced Citation source blob readiness", + "target", + lambda: _preflight_target_blob_migration_access( + normalized_settings, + normalized_plan, + ), + "Selected source and destination Blob containers are ready.", + )) + else: + checks.append({ + "id": "source_blob_access", + "label": "Enhanced Citation source blob readiness", + "workflow_step": "options", + "status": "warning", + "summary": "Enhanced Citation source blobs are excluded from this migration.", + "details": None, + }) + + if normalized_settings.get("migration_temporary_destination_ru_enabled"): + checks.append(_migration_review_check( + "destination_capacity", + "Destination capacity policy", + "options", + lambda: _inspect_target_cosmos_migration_capacity( + normalized_settings, + normalized_plan, + ), + "Temporary destination capacity can be inspected with the configured policy.", + )) + else: + checks.append({ + "id": "destination_capacity", + "label": "Destination capacity policy", + "workflow_step": "options", + "status": "pass", + "summary": "Migration will use the destination's current Cosmos capacity.", + "details": { + "temporary_capacity_enabled": False, + }, + }) + + preview_check = _migration_review_check( + "destination_inventory", + "Destination inventory and collisions", + "review", + lambda: preview_data_management_migration_plan( + normalized_settings, + {"migration_plan": normalized_plan}, + resolved_migration_plan=normalized_plan, + ), + "Destination inventory completed without detected collisions.", + ) + preview = preview_check.get("details") + estimated_outcomes = ( + preview.get("estimated_outcomes") + if isinstance(preview, dict) + else {} + ) or {} + if ( + preview_check["status"] == "pass" and + _safe_int(estimated_outcomes.get("conflict_count"), default=0) > 0 + ): + preview_check.update({ + "status": "block", + "summary": "Destination collisions must be resolved before execution.", + }) + elif ( + preview_check["status"] == "pass" and + _safe_int(estimated_outcomes.get("delete_count"), default=0) > 0 + ): + preview_check.update({ + "status": "warning", + "summary": "The reviewed mirror plan includes destination deletions.", + }) + checks.append(preview_check) + + blockers = sum(check["status"] == "block" for check in checks) + warnings = sum(check["status"] == "warning" for check in checks) + return { + "reviewed_at": _now_iso(), + "review_fingerprint": _migration_review_fingerprint( + normalized_settings, + normalized_plan, + ), + "ready": blockers == 0, + "blocker_count": blockers, + "warning_count": warnings, + "summary": summary, + "preview": preview, + "checks": checks, + "execution": { + "migration_mode": normalized_plan.get("migration_mode"), + "max_parallel_operations": normalized_settings.get( + "migration_max_parallel_operations" + ), + "retry_count": normalized_settings.get("migration_retry_count"), + "skip_recent_within_hours": normalized_settings.get( + "migration_skip_recent_within_hours" + ), + "temporary_destination_ru_enabled": normalized_settings.get( + "migration_temporary_destination_ru_enabled" + ), + "temporary_destination_ru": normalized_settings.get( + "migration_temporary_destination_ru" + ), + }, + } + + def _count_preview_source_cosmos_records(migration_plan, source_cutoff_at): source_cutoff = _parse_iso_datetime(source_cutoff_at) source_cutoff_epoch = int(source_cutoff.timestamp()) if source_cutoff else None @@ -14840,6 +15677,33 @@ def execute_backup_job(job, settings): def execute_migration_job(job, settings): options = job.get("options") if isinstance(job.get("options"), dict) else {} migration_plan = normalize_data_management_migration_plan(options) + reviewed_fingerprint = _safe_text(options.get("review_fingerprint")) + if reviewed_fingerprint: + current_fingerprint = _migration_review_fingerprint( + settings, + migration_plan, + ) + if not hmac.compare_digest( + reviewed_fingerprint, + current_fingerprint, + ): + raise DataManagementSettingsValidationError( + "Migration settings changed after review. Run preflight review again." + ) + review_authorization_token = _safe_text( + options.get("review_authorization_token") + ) + review_reservation_token = _safe_text( + options.get("review_reservation_token") + ) + if review_authorization_token and review_reservation_token: + consume_data_management_migration_review_authorization( + review_authorization_token, + job.get("requested_by"), + reviewed_fingerprint, + review_reservation_token, + job.get("id"), + ) total_steps = 10 _set_job_progress( job, diff --git a/application/single_app/functions_data_management_search_write_fence.py b/application/single_app/functions_data_management_search_write_fence.py index cb246230e..3e83b24e2 100644 --- a/application/single_app/functions_data_management_search_write_fence.py +++ b/application/single_app/functions_data_management_search_write_fence.py @@ -459,6 +459,36 @@ def _target_migration_coordinator_is_active(coordinator, now=None): ) +def inspect_data_management_target_migration_coordinator(container): + """Return secret-free destination coordinator readiness.""" + coordinator = _read_target_migration_coordinator(container) + active = _target_migration_coordinator_is_active(coordinator) + return { + "available": not active, + "active": active, + "expires_at": coordinator.get("expires_at") if active else None, + } + + +def inspect_data_management_search_write_gate(container): + """Return secret-free Search write-gate readiness.""" + gate = _read_gate(container) + if gate is None: + return { + "available": True, + "state": DATA_MANAGEMENT_SEARCH_WRITE_GATE_STATE_OPEN, + "active_writer_count": 0, + "expires_at": None, + } + active_fence = _is_active_migration_fence(gate) + return { + "available": not active_fence, + "state": gate.get("state") or DATA_MANAGEMENT_SEARCH_WRITE_GATE_STATE_OPEN, + "active_writer_count": len(_active_writer_leases(gate)), + "expires_at": gate.get("expires_at") if active_fence else None, + } + + def _new_target_migration_coordinator(migration_id, lock_token, lease_seconds, now=None): timestamp = now or _now_utc() return { diff --git a/application/single_app/route_backend_data_management.py b/application/single_app/route_backend_data_management.py index 691064cd3..6e8aacb86 100644 --- a/application/single_app/route_backend_data_management.py +++ b/application/single_app/route_backend_data_management.py @@ -1,6 +1,7 @@ # route_backend_data_management.py """Admin API routes for SimpleChat Data Management.""" +import hmac import logging from flask import Response, current_app, jsonify, request, session @@ -16,6 +17,7 @@ DataManagementCosmosEditorError, DataManagementHistoryPaginationError, DataManagementSettingsValidationError, + create_data_management_migration_review_authorization, export_data_management_migration_manifest, generate_data_management_encryption_key, get_data_management_cosmos_editor_containers, @@ -25,12 +27,16 @@ get_data_management_job_progress, get_data_management_jobs_page, get_data_management_migration_catalog, + get_data_management_migration_review_fingerprint, get_data_management_settings, log_data_management_cosmos_editor_activity, preview_data_management_migration_plan, queue_data_management_job, query_data_management_cosmos_editor_documents, request_data_management_job_cancellation, + release_data_management_migration_review_reservation, + reserve_data_management_migration_review_authorization, + review_data_management_migration, retry_data_management_backup_job, resolve_data_management_migration_manifest_item, retry_data_management_migration_job, @@ -596,9 +602,15 @@ def list_admin_data_management_backups(): @admin_required def get_admin_data_management_migration_catalog(target_type): search = request.args.get("search", "") - limit = request.args.get("limit", 50) + limit = request.args.get("page_size", request.args.get("limit", 25)) + continuation_token = request.args.get("continuation_token", "") try: - catalog = get_data_management_migration_catalog(target_type, search_text=search, limit=limit) + catalog = get_data_management_migration_catalog( + target_type, + search_text=search, + limit=limit, + continuation_token=continuation_token, + ) except DataManagementSettingsValidationError as exc: return jsonify({"success": False, "error": str(exc)}), 400 return jsonify({"success": True, **catalog}), 200 @@ -632,6 +644,59 @@ def summarize_admin_data_management_migration(): }), 400 return jsonify({"success": True, "summary": summary, "preview": preview}), 200 + @bp.route("/api/admin/data-management/migration/review", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def review_admin_data_management_migration(): + payload = request.get_json(silent=True) or {} + try: + review = review_data_management_migration( + settings=( + payload.get("settings") + if isinstance(payload.get("settings"), dict) + else None + ), + migration_plan=( + payload.get("migration_plan") + if isinstance(payload.get("migration_plan"), dict) + else None + ), + ) + if review.get("ready") is True: + admin_user_id, _admin_email = _get_admin_context() + review.update( + create_data_management_migration_review_authorization( + admin_user_id, + review.get("review_fingerprint"), + ) + ) + except DataManagementSettingsValidationError as exc: + log_event( + "[DataManagement] Migration review validation failed.", + {"error_type": type(exc).__name__}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return jsonify({ + "success": False, + "error": "Migration review input is invalid.", + "workflow_step": "review", + }), 400 + except Exception as exc: + log_event( + "[DataManagement] Migration review failed.", + {"error_type": type(exc).__name__}, + level=logging.ERROR, + exceptionTraceback=True, + ) + return jsonify({ + "success": False, + "error": "Migration review could not be completed.", + "workflow_step": "review", + }), 400 + return jsonify({"success": True, "review": review}), 200 + @bp.route("/api/admin/data-management/jobs", methods=["POST"]) @swagger_route(security=get_auth_security()) @login_required @@ -640,6 +705,7 @@ def create_admin_data_management_job(): payload = request.get_json(silent=True) or {} operation = str(payload.get("operation") or DATA_MANAGEMENT_OPERATION_DRY_RUN).strip() backup_type = payload.get("backup_type") + review_reservation = None if operation not in { DATA_MANAGEMENT_OPERATION_BACKUP, DATA_MANAGEMENT_OPERATION_RESTORE, @@ -655,14 +721,96 @@ def create_admin_data_management_job(): if isinstance(payload.get("options"), dict) else {} ) - job = queue_data_management_job( - operation, - backup_type=backup_type, - requested_by=admin_user_id, - requested_by_email=admin_email, - options=job_options, - ) + if operation == DATA_MANAGEMENT_OPERATION_MIGRATION: + provided_review_fingerprint = str( + job_options.get("review_fingerprint") or "" + ).strip() + review_authorization_token = str( + job_options.get("review_authorization_token") or "" + ).strip() + expected_review_fingerprint = ( + get_data_management_migration_review_fingerprint( + migration_plan=( + job_options.get("migration_plan") + if isinstance( + job_options.get("migration_plan"), + dict, + ) + else None + ), + ) + ) + if ( + not provided_review_fingerprint or + not hmac.compare_digest( + provided_review_fingerprint, + expected_review_fingerprint, + ) + ): + return jsonify({ + "success": False, + "error": ( + "Migration inputs changed after preflight review. " + "Run review again before execution." + ), + "workflow_step": "review", + }), 409 + try: + review_reservation = ( + reserve_data_management_migration_review_authorization( + review_authorization_token, + admin_user_id, + expected_review_fingerprint, + ) + ) + job_options["review_reservation_token"] = ( + review_reservation["reservation_token"] + ) + except DataManagementSettingsValidationError as exc: + log_event( + "[DataManagement] Migration review reservation validation failed.", + {"operation": operation, "error": str(exc)}, + level=logging.WARNING, + ) + return jsonify({ + "success": False, + "error": "Migration review authorization is invalid or expired. Run review again before execution.", + "workflow_step": "review", + }), 409 + try: + job = queue_data_management_job( + operation, + backup_type=backup_type, + requested_by=admin_user_id, + requested_by_email=admin_email, + options=job_options, + occurrence_id=( + review_reservation.get("job_id") + if review_reservation + else None + ), + ) + except Exception: + if review_reservation: + release_data_management_migration_review_reservation( + review_authorization_token, + review_reservation["reservation_token"], + ) + raise submitted = submit_data_management_job(current_app._get_current_object(), job.get("id")) + except DataManagementSettingsValidationError as exc: + log_event( + "[DataManagement] Validation error while queuing data management job.", + {"operation": operation, "error": str(exc)}, + level=logging.WARNING, + ) + response = { + "success": False, + "error": "Data Management job request is invalid.", + } + if operation == DATA_MANAGEMENT_OPERATION_MIGRATION: + response["workflow_step"] = "confirm" + return jsonify(response), 400 except Exception as exc: log_event( "[DataManagement] Failed to queue data management job.", diff --git a/application/single_app/static/css/styles.css b/application/single_app/static/css/styles.css index 5bb96240a..65230c0fa 100644 --- a/application/single_app/static/css/styles.css +++ b/application/single_app/static/css/styles.css @@ -1261,6 +1261,554 @@ main { color: #adb5bd; } +/* Admin Data Management migration workflow */ +.migration-workflow { + border-color: var(--bs-border-color); + overflow: hidden; +} + +.migration-workflow-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + padding: 1.25rem 1.5rem; + background-color: var(--bs-tertiary-bg); + border-bottom: 1px solid var(--bs-border-color); +} + +.migration-workflow-header p { + max-width: 72ch; + color: var(--bs-secondary-color); +} + +.migration-workflow-readiness { + display: inline-flex; + align-items: center; + gap: 0.5rem; + flex: 0 0 auto; + padding: 0.4rem 0.75rem; + border: 1px solid var(--bs-border-color); + border-radius: 999px; + background-color: var(--bs-body-bg); + color: var(--bs-secondary-color); + font-size: 0.875rem; + font-weight: 600; +} + +.migration-readiness-dot { + width: 0.625rem; + height: 0.625rem; + border-radius: 50%; + background-color: var(--bs-secondary-color); +} + +.migration-workflow-readiness.is-ready { + color: var(--bs-success-text-emphasis); + border-color: var(--bs-success-border-subtle); + background-color: var(--bs-success-bg-subtle); +} + +.migration-workflow-readiness.is-ready .migration-readiness-dot { + background-color: var(--bs-success); +} + +.migration-workflow-readiness.is-blocked { + color: var(--bs-danger-text-emphasis); + border-color: var(--bs-danger-border-subtle); + background-color: var(--bs-danger-bg-subtle); +} + +.migration-workflow-readiness.is-blocked .migration-readiness-dot { + background-color: var(--bs-danger); +} + +.migration-workflow-readiness.is-stale { + color: var(--bs-warning-text-emphasis); + border-color: var(--bs-warning-border-subtle); + background-color: var(--bs-warning-bg-subtle); +} + +.migration-workflow-readiness.is-stale .migration-readiness-dot { + background-color: var(--bs-warning); +} + +.migration-workflow-body { + padding: 0 1.5rem 1.5rem; +} + +.migration-stepper { + display: grid; + grid-template-columns: repeat(6, minmax(8rem, 1fr)); + margin: 0 -1.5rem; + overflow-x: auto; + border-bottom: 1px solid var(--bs-border-color); + background-color: var(--bs-body-bg); +} + +.migration-step { + display: flex; + align-items: center; + gap: 0.65rem; + min-width: 0; + padding: 1rem 0.75rem; + color: var(--bs-secondary-color); + text-align: left; + background: transparent; + border: 0; + border-bottom: 3px solid transparent; +} + +.migration-step:hover:not(:disabled) { + color: var(--bs-body-color); + background-color: var(--bs-tertiary-bg); +} + +.migration-step:focus-visible { + position: relative; + z-index: 1; + outline: 3px solid rgba(var(--bs-primary-rgb), 0.35); + outline-offset: -3px; +} + +.migration-step.is-active { + color: var(--bs-primary-text-emphasis); + border-bottom-color: var(--bs-primary); + background-color: var(--bs-primary-bg-subtle); +} + +.migration-step.is-complete { + color: var(--bs-success-text-emphasis); +} + +.migration-step.has-error { + color: var(--bs-danger-text-emphasis); + border-bottom-color: var(--bs-danger); +} + +.migration-step-marker { + display: inline-grid; + place-items: center; + width: 1.75rem; + height: 1.75rem; + flex: 0 0 auto; + border: 1px solid var(--bs-border-color); + border-radius: 50%; + background-color: var(--bs-body-bg); + font-size: 0.8rem; + font-weight: 700; +} + +.migration-step.is-active .migration-step-marker { + color: var(--bs-white); + border-color: var(--bs-primary); + background-color: var(--bs-primary); +} + +.migration-step.is-complete .migration-step-marker { + color: var(--bs-white); + border-color: var(--bs-success); + background-color: var(--bs-success); +} + +.migration-step.has-error .migration-step-marker { + color: var(--bs-white); + border-color: var(--bs-danger); + background-color: var(--bs-danger); +} + +.migration-step-copy { + display: flex; + flex-direction: column; + min-width: 0; +} + +.migration-step-label { + overflow: hidden; + font-size: 0.875rem; + font-weight: 700; + text-overflow: ellipsis; + white-space: nowrap; +} + +.migration-step-state { + overflow: hidden; + font-size: 0.75rem; + text-overflow: ellipsis; + white-space: nowrap; +} + +.migration-stage { + min-height: 34rem; + padding: 1.75rem 0 1rem; +} + +.migration-stage-heading { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 1.5rem; +} + +.migration-stage-heading h5 { + margin-bottom: 0.35rem; +} + +.migration-stage-heading p { + max-width: 72ch; + margin-bottom: 0; + color: var(--bs-secondary-color); +} + +.migration-target-grid { + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius-lg); +} + +.migration-service-panel { + padding: 1.25rem; +} + +.migration-service-panel + .migration-service-panel { + border-top: 1px solid var(--bs-border-color); +} + +.migration-service-heading { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 1rem; +} + +.migration-service-heading h6 { + margin-bottom: 0.25rem; +} + +.migration-service-heading p { + margin-bottom: 0; + color: var(--bs-secondary-color); + font-size: 0.875rem; +} + +.migration-scope-total { + padding: 0.5rem 0.75rem; + border-radius: var(--bs-border-radius); + background-color: var(--bs-tertiary-bg); + color: var(--bs-secondary-color); + font-size: 0.875rem; + font-weight: 600; +} + +.migration-scope-tabs { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + margin-bottom: 1.25rem; +} + +.migration-scope-tabs .btn.active { + color: var(--bs-white); + background-color: var(--bs-primary); + border-color: var(--bs-primary); +} + +.migration-scope-pane { + padding: 1.25rem; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius-lg); + background-color: var(--bs-tertiary-bg); +} + +.migration-mode-options { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 0.75rem; +} + +.migration-mode-option { + position: relative; + cursor: pointer; +} + +.migration-mode-option input { + position: absolute; + opacity: 0; + pointer-events: none; +} + +.migration-mode-option span { + display: flex; + flex-direction: column; + min-height: 4.5rem; + padding: 0.85rem 1rem; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius); + background-color: var(--bs-body-bg); +} + +.migration-mode-option small { + margin-top: 0.2rem; + color: var(--bs-secondary-color); +} + +.migration-mode-option input:checked + span { + color: var(--bs-primary-text-emphasis); + border-color: var(--bs-primary); + background-color: var(--bs-primary-bg-subtle); + box-shadow: 0 0.25rem 0.75rem rgba(var(--bs-primary-rgb), 0.12); +} + +.migration-mode-option input:focus-visible + span { + outline: 3px solid rgba(var(--bs-primary-rgb), 0.35); + outline-offset: 2px; +} + +.migration-catalog-workspace { + display: grid; + grid-template-columns: minmax(0, 1.6fr) minmax(16rem, 0.8fr); + gap: 1rem; + margin-top: 1rem; +} + +.migration-catalog-column, +.migration-selection-column { + min-width: 0; + padding: 1rem; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius); + background-color: var(--bs-body-bg); +} + +.migration-catalog-search { + width: min(100%, 24rem); +} + +.migration-catalog-list, +.migration-selected-list { + min-height: 14rem; + max-height: 22rem; + overflow-y: auto; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius); +} + +.migration-catalog-pagination { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.75rem; + margin-top: 0.75rem; +} + +.migration-all-summary, +.migration-review-empty, +.migration-progress-empty { + display: flex; + align-items: flex-start; + gap: 0.9rem; + margin-top: 1rem; + padding: 1.15rem; + border: 1px solid var(--bs-info-border-subtle); + border-radius: var(--bs-border-radius); + background-color: var(--bs-info-bg-subtle); + color: var(--bs-info-text-emphasis); +} + +.migration-all-summary > i, +.migration-review-empty > i, +.migration-progress-empty > i { + font-size: 1.4rem; +} + +.migration-all-summary div, +.migration-review-empty div, +.migration-progress-empty div { + display: flex; + flex-direction: column; + gap: 0.2rem; +} + +.migration-option-section { + margin: 0; + padding: 1.25rem 0; + border: 0; + border-top: 1px solid var(--bs-border-color); +} + +.migration-option-section:first-of-type { + padding-top: 0; + border-top: 0; +} + +.migration-option-section legend { + float: none; + width: auto; + margin-bottom: 0.75rem; + font-size: 1rem; + font-weight: 700; +} + +.migration-review-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 0.75rem; +} + +.migration-review-metric { + padding: 1rem; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius); + background-color: var(--bs-tertiary-bg); +} + +.migration-review-metric span { + display: block; + color: var(--bs-secondary-color); + font-size: 0.8rem; +} + +.migration-review-metric strong { + display: block; + margin-top: 0.2rem; + font-size: 1.15rem; +} + +.migration-review-checks { + margin-top: 1rem; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius-lg); + overflow: hidden; +} + +.migration-review-check { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + align-items: start; + gap: 0.8rem; + padding: 1rem; +} + +.migration-review-check + .migration-review-check { + border-top: 1px solid var(--bs-border-color); +} + +.migration-review-check i { + font-size: 1.1rem; +} + +.migration-review-check p { + margin: 0.2rem 0 0; + color: var(--bs-secondary-color); + font-size: 0.875rem; +} + +.migration-review-check.is-pass i { + color: var(--bs-success); +} + +.migration-review-check.is-warning i { + color: var(--bs-warning-text-emphasis); +} + +.migration-review-check.is-block i { + color: var(--bs-danger); +} + +.migration-confirm-summary { + padding: 1.25rem; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius-lg); + background-color: var(--bs-tertiary-bg); +} + +.migration-workflow-footer { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + padding-top: 1rem; + border-top: 1px solid var(--bs-border-color); +} + +@media (max-width: 991.98px) { + .migration-stepper { + grid-auto-flow: column; + grid-auto-columns: minmax(9rem, 1fr); + grid-template-columns: none; + } + + .migration-catalog-workspace { + grid-template-columns: 1fr; + } + + .migration-review-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +@media (max-width: 767.98px) { + .migration-workflow-header, + .migration-stage-heading, + .migration-service-heading, + .migration-workflow-footer { + align-items: stretch; + flex-direction: column; + } + + .migration-workflow-header, + .migration-workflow-body { + padding-right: 1rem; + padding-left: 1rem; + } + + .migration-stepper { + margin-right: -1rem; + margin-left: -1rem; + } + + .migration-workflow-readiness { + align-self: flex-start; + } + + .migration-stage { + min-height: 0; + padding-top: 1.25rem; + } + + .migration-mode-options, + .migration-review-grid { + grid-template-columns: 1fr; + } + + .migration-catalog-search { + width: 100%; + } + + .migration-review-check { + grid-template-columns: auto minmax(0, 1fr); + } + + .migration-review-check .badge { + grid-column: 2; + justify-self: start; + } + + .migration-workflow-footer > div { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.5rem; + } +} + +@media (prefers-reduced-motion: reduce) { + .migration-step, + .migration-mode-option span { + transition: none; + } +} + /* Cosmos editor modal */ #data-management-cosmos-editor-results-modal .cosmos-editor-results-modal-body { overflow-x: hidden; diff --git a/application/single_app/static/js/admin/admin_data_management.js b/application/single_app/static/js/admin/admin_data_management.js index 277d1e253..f28b18178 100644 --- a/application/single_app/static/js/admin/admin_data_management.js +++ b/application/single_app/static/js/admin/admin_data_management.js @@ -23,6 +23,8 @@ let cosmosEditorResultCount = 0; let cosmosEditorSelectedDocument = null; let cosmosEditorPendingDocument = null; let pendingDataManagementCancellationJob = null; +let migrationWorkflowRefreshTimer = null; +let migrationWorkflowRefreshInFlight = false; const jobDetailRefreshIntervalMs = 2000; const activeJobStatuses = new Set(["queued", "running"]); @@ -31,11 +33,36 @@ const historyListState = { jobs: createHistoryListState(), }; const migrationTargetTypes = ["users", "groups", "public_workspaces"]; +const migrationWorkflowSteps = ["target", "scope", "options", "review", "confirm", "progress"]; const migrationSelections = { users: new Map(), groups: new Map(), public_workspaces: new Map(), }; +const migrationCatalogStates = Object.fromEntries( + migrationTargetTypes.map((targetType) => [ + targetType, + { + items: [], + totalCount: 0, + pageIndex: 0, + pageTokens: [""], + nextToken: "", + requestGeneration: 0, + search: "", + }, + ]) +); +const migrationWorkflowState = { + activeStepIndex: 0, + highestVisitedStepIndex: 0, + review: null, + reviewStale: true, + reviewRequestGeneration: 0, + submissionInFlight: false, + submissionAccepted: false, + currentJob: null, +}; document.addEventListener("DOMContentLoaded", () => { bindElements(); @@ -44,6 +71,7 @@ document.addEventListener("DOMContentLoaded", () => { } bindEvents(); + initializeMigrationWorkflow(); loadDataManagementSettings(); loadDataManagementBackups(); loadDataManagementJobs(); @@ -54,6 +82,24 @@ document.addEventListener("DOMContentLoaded", () => { function bindElements() { const ids = [ "data-management", + "data-management-migration-section", + "data-management-migration-readiness", + "data-management-migration-step-error", + "data-management-migration-scope-total", + "data-management-migration-review-empty", + "data-management-migration-review-checks", + "data-management-migration-confirm-summary", + "data_management_migration_final_confirmation", + "data-management-migration-start-new-btn", + "data-management-migration-view-job-btn", + "data-management-migration-progress-empty", + "data-management-migration-workflow-progress", + "data-management-migration-workflow-progress-summary", + "data-management-migration-workflow-progress-state", + "data-management-migration-workflow-progress-actions", + "data-management-migration-back-btn", + "data-management-migration-next-btn", + "data-management-migration-step-position", "data-management-status", "data-management-save-settings-btn", "data_management_enabled", @@ -265,10 +311,11 @@ function bindEvents() { elements.dataManagementTestTargetEcStorageBtn?.addEventListener("click", testTargetEnhancedCitationStorage); elements.dataManagementRunFullBackupBtn?.addEventListener("click", () => queueBackup("full")); elements.dataManagementRunPartialBackupBtn?.addEventListener("click", () => queueBackup("partial")); - elements.dataManagementMigrationPreviewBtn?.addEventListener("click", () => loadMigrationSummary(elements.dataManagementMigrationPreviewBtn, true)); + elements.dataManagementMigrationPreviewBtn?.addEventListener("click", () => runMigrationReview(elements.dataManagementMigrationPreviewBtn)); elements.dataManagementExecuteMigrationBtn?.addEventListener("click", () => queueMigration(false)); - elements.dataManagementRefreshMigrationSummaryBtn?.addEventListener("click", () => loadMigrationSummary(elements.dataManagementRefreshMigrationSummaryBtn, true)); + elements.dataManagementRefreshMigrationSummaryBtn?.addEventListener("click", () => runMigrationReview(elements.dataManagementRefreshMigrationSummaryBtn)); bindMigrationPickerEvents(); + bindMigrationWorkflowEvents(); elements.dataManagementRefreshBackupsBtn?.addEventListener("click", loadDataManagementBackups); elements.dataManagementViewFullBackupsBtn?.addEventListener("click", () => setBackupFilter("full", { load: true })); elements.dataManagementViewPartialBackupsBtn?.addEventListener("click", () => setBackupFilter("partial", { load: true })); @@ -318,12 +365,14 @@ function bindEvents() { elements.datamanagementmigrationmodemirrorwithdeletions, ].forEach((modeElement) => modeElement?.addEventListener("change", () => { updateMigrationModeVisibility(); - loadMigrationSummary(); + invalidateMigrationReview(); })); - elements.datamanagementmigrationbaselinejobid?.addEventListener("change", loadMigrationSummary); + elements.datamanagementmigrationbaselinejobid?.addEventListener("change", invalidateMigrationReview); elements.datamanagementmigrationmirrorconfirmationphrase?.addEventListener("input", () => { elements.datamanagementmigrationmirrorconfirmationphrase?.classList.remove("is-invalid"); + updateMigrationExecutionButtonState(); }); + elements.datamanagementmigrationfinalconfirmation?.addEventListener("change", updateMigrationExecutionButtonState); bindDataManagementChangeTracking(); setStorageAuthVisibility(); setMigrationTargetVisibility(); @@ -337,24 +386,122 @@ function bindEvents() { function bindMigrationPickerEvents() { migrationTargetTypes.forEach((targetType) => { - getMigrationModeElement(targetType)?.addEventListener("change", () => { - updateMigrationPickerVisibility(targetType); - loadMigrationSummary(); + document.querySelectorAll(`[data-migration-mode-option="${targetType}"]`).forEach((modeOption) => { + modeOption.addEventListener("change", () => { + if (!modeOption.checked) { + return; + } + setMigrationScopeMode(targetType, modeOption.value); + }); }); - getMigrationDocumentsElement(targetType)?.addEventListener("change", loadMigrationSummary); - getMigrationSearchButton(targetType)?.addEventListener("click", () => loadMigrationCatalog(targetType)); + getMigrationDocumentsElement(targetType)?.addEventListener("change", () => { + invalidateMigrationReview(); + updateMigrationScopeCounts(); + }); + getMigrationSearchButton(targetType)?.addEventListener("click", () => loadMigrationCatalog(targetType, "reset")); getMigrationSearchElement(targetType)?.addEventListener("keydown", (event) => { if (event.key === "Enter") { event.preventDefault(); - loadMigrationCatalog(targetType); + loadMigrationCatalog(targetType, "reset"); } }); + document.querySelector(`[data-migration-page-previous="${targetType}"]`)?.addEventListener("click", () => loadMigrationCatalog(targetType, "previous")); + document.querySelector(`[data-migration-page-next="${targetType}"]`)?.addEventListener("click", () => loadMigrationCatalog(targetType, "next")); }); elements.datamanagementmigrationincludeaisearch?.addEventListener("change", () => { updateMigrationSearchWriteFreezeVisibility(); - loadMigrationSummary(); + invalidateMigrationReview(); + }); + elements.datamanagementmigrationincludesourceblobs?.addEventListener("change", invalidateMigrationReview); +} + +function bindMigrationWorkflowEvents() { + elements.dataManagementMigrationBackBtn?.addEventListener("click", () => { + showMigrationWorkflowStep(migrationWorkflowState.activeStepIndex - 1); + }); + elements.dataManagementMigrationNextBtn?.addEventListener("click", advanceMigrationWorkflow); + elements.dataManagementMigrationViewJobBtn?.addEventListener("click", () => { + const jobId = migrationWorkflowState.currentJob?.id; + if (jobId) { + loadDataManagementJobDetail(jobId); + } + }); + elements.dataManagementMigrationStartNewBtn?.addEventListener("click", resetMigrationWorkflowForNewRun); + document.querySelectorAll("[data-migration-step-button]").forEach((button) => { + button.addEventListener("click", () => { + const stepIndex = migrationWorkflowSteps.indexOf(button.dataset.migrationStepButton); + if (stepIndex < 0) { + return; + } + const canVisit = ( + stepIndex <= migrationWorkflowState.highestVisitedStepIndex || + (button.dataset.migrationStepButton === "confirm" && isMigrationReviewCurrentAndReady()) || + (button.dataset.migrationStepButton === "progress" && Boolean(migrationWorkflowState.currentJob)) + ); + if (canVisit) { + showMigrationWorkflowStep(stepIndex); + } + }); + }); + document.querySelectorAll("[data-migration-scope-tab]").forEach((button) => { + button.addEventListener("click", () => activateMigrationScopeTab(button.dataset.migrationScopeTab)); + }); + elements.dataManagementMigrationSection?.addEventListener("input", handleMigrationWorkflowInput); + elements.dataManagementMigrationSection?.addEventListener("change", handleMigrationWorkflowInput); +} + +function initializeMigrationWorkflow() { + migrationTargetTypes.forEach((targetType) => { + setValue(getMigrationModeElement(targetType), "none"); + updateMigrationPickerVisibility(targetType); + renderMigrationSelectedList(targetType); + }); + activateMigrationScopeTab("users"); + updateMigrationScopeCounts(); + updateMigrationReadiness(); + showMigrationWorkflowStep(0, { skipValidation: true }); +} + +function handleMigrationWorkflowInput(event) { + const target = event.target; + if (!(target instanceof HTMLElement)) { + return; + } + if ( + target === elements.datamanagementmigrationfinalconfirmation || + target === elements.datamanagementmigrationmirrorconfirmationphrase || + target.matches("input[type='search']") + ) { + return; + } + invalidateMigrationReview(); +} + +function setMigrationScopeMode(targetType, mode) { + const normalizedMode = ["none", "selected", "all"].includes(mode) ? mode : "none"; + setValue(getMigrationModeElement(targetType), normalizedMode); + updateMigrationPickerVisibility(targetType); + if (normalizedMode === "selected" && migrationCatalogStates[targetType].items.length === 0) { + loadMigrationCatalog(targetType, "reset"); + } else if (normalizedMode === "all") { + loadMigrationCatalog(targetType, "count"); + } + invalidateMigrationReview(); + updateMigrationScopeCounts(); +} + +function activateMigrationScopeTab(targetType) { + if (!migrationTargetTypes.includes(targetType)) { + return; + } + document.querySelectorAll("[data-migration-scope-tab]").forEach((button) => { + const isActive = button.dataset.migrationScopeTab === targetType; + button.classList.toggle("active", isActive); + button.setAttribute("aria-selected", String(isActive)); + }); + document.querySelectorAll("[data-migration-scope-pane]").forEach((pane) => { + pane.classList.toggle("d-none", pane.dataset.migrationScopePane !== targetType); }); - elements.datamanagementmigrationincludesourceblobs?.addEventListener("change", loadMigrationSummary); } function bindDataManagementChangeTracking() { @@ -763,7 +910,9 @@ async function requestJson(url, options = {}) { } const data = await response.json().catch(() => ({})); if (!response.ok || data.success === false) { - throw new Error(data.error || `Request failed with status ${response.status}`); + const requestError = new Error(data.error || `Request failed with status ${response.status}`); + requestError.workflowStep = data.workflow_step || ""; + throw requestError; } return data; } @@ -1407,15 +1556,55 @@ async function queueOperation(operation, backupType = null, options = {}, trigge renderJobs([data.job]); loadDataManagementJobs(); loadDataManagementBackups(); + return data; } catch (error) { setStatus(error.message || "Data Management job could not be queued.", "danger"); showToast(error.message || "Data Management job could not be queued.", "danger"); + if (operation === "migration") { + const workflowStep = error.workflowStep || "confirm"; + if (workflowStep === "review") { + migrationWorkflowState.reviewStale = true; + migrationWorkflowState.reviewRequestGeneration += 1; + setChecked( + elements.datamanagementmigrationfinalconfirmation, + false + ); + updateMigrationReadiness(); + updateMigrationExecutionButtonState(); + updateMigrationWorkflowFooter(); + showMigrationWorkflowStep( + migrationWorkflowSteps.indexOf("review"), + { skipValidation: true } + ); + } + showMigrationStepError( + error.message || "Migration could not be queued.", + workflowStep + ); + } + return null; } finally { setBusy(button, false); } } -function queueMigration(dryRun) { +async function queueMigration(dryRun) { + if ( + migrationWorkflowState.submissionInFlight || + migrationWorkflowState.submissionAccepted + ) { + return; + } + if (!isMigrationReviewCurrentAndReady()) { + showMigrationStepError("Run a current preflight review and resolve every blocker before execution.", "review"); + showMigrationWorkflowStep(migrationWorkflowSteps.indexOf("review"), { skipValidation: true }); + return; + } + if (!elements.datamanagementmigrationfinalconfirmation?.checked) { + showMigrationStepError("Confirm that you reviewed the normalized plan and operational impact.", "confirm"); + elements.datamanagementmigrationfinalconfirmation?.focus(); + return; + } const button = elements.dataManagementExecuteMigrationBtn; const migrationPlan = buildMigrationPlan(); if ( @@ -1426,7 +1615,7 @@ function queueMigration(dryRun) { elements.datamanagementmigrationtargetsearchwritesfrozen?.focus(); setStatus(message, "danger"); showToast(message, "danger"); - return Promise.resolve(); + return; } if ( migrationPlan.migration_mode === "mirror_with_deletions" && @@ -1437,12 +1626,30 @@ function queueMigration(dryRun) { elements.datamanagementmigrationmirrorconfirmationphrase?.focus(); setStatus(message, "danger"); showToast(message, "danger"); - return Promise.resolve(); + return; + } + migrationWorkflowState.submissionInFlight = true; + updateMigrationExecutionButtonState(); + try { + const data = await queueOperation("migration", null, { + dry_run: Boolean(dryRun), + migration_plan: migrationPlan, + review_fingerprint: migrationWorkflowState.review?.review_fingerprint || "", + review_authorization_token: migrationWorkflowState.review?.authorization_token || "", + }, button); + if (!data?.job) { + return; + } + migrationWorkflowState.submissionAccepted = true; + if (elements.datamanagementmigrationfinalconfirmation) { + elements.datamanagementmigrationfinalconfirmation.checked = false; + } + attachMigrationWorkflowJob(data.job); + showMigrationWorkflowStep(migrationWorkflowSteps.indexOf("progress"), { skipValidation: true }); + } finally { + migrationWorkflowState.submissionInFlight = false; + updateMigrationExecutionButtonState(); } - return queueOperation("migration", null, { - dry_run: Boolean(dryRun), - migration_plan: migrationPlan, - }, button); } function buttonForOperation(operation, backupType) { @@ -1493,29 +1700,84 @@ function formatMigrationTargetName(targetType) { function updateMigrationPickerVisibility(targetType) { const mode = getValue(getMigrationModeElement(targetType)) || "none"; const isSelectedMode = mode === "selected"; - const searchElement = getMigrationSearchElement(targetType); - const searchButton = getMigrationSearchButton(targetType); - if (searchElement) { - searchElement.disabled = !isSelectedMode; - } - if (searchButton) { - searchButton.disabled = !isSelectedMode; + const isAllMode = mode === "all"; + const workspace = document.querySelector(`[data-migration-catalog-workspace="${targetType}"]`); + const allSummary = document.querySelector(`[data-migration-all-summary="${targetType}"]`); + setElementVisible(workspace, isSelectedMode); + setElementVisible(allSummary, isAllMode); + const documentsElement = getMigrationDocumentsElement(targetType); + if (documentsElement) { + documentsElement.disabled = mode === "none"; + if (mode === "none") { + documentsElement.checked = false; + } } renderMigrationSelectedList(targetType); } -async function loadMigrationCatalog(targetType) { +async function loadMigrationCatalog(targetType, direction = "reset") { const availableElement = getMigrationAvailableElement(targetType); - if (!availableElement) { + const state = migrationCatalogStates[targetType]; + if (!availableElement || !state) { + return; + } + const countOnly = direction === "count"; + if (direction === "reset") { + state.search = getValue(getMigrationSearchElement(targetType)); + state.pageIndex = 0; + state.pageTokens = [""]; + state.nextToken = ""; + } else if (direction === "next" && state.nextToken) { + state.pageIndex += 1; + state.pageTokens = state.pageTokens.slice(0, state.pageIndex); + state.pageTokens[state.pageIndex] = state.nextToken; + } else if (direction === "previous" && state.pageIndex > 0) { + state.pageIndex -= 1; + } else if (!countOnly) { return; } - availableElement.replaceChildren(createSmallMutedElement("Loading...")); + + const requestGeneration = state.requestGeneration + 1; + state.requestGeneration = requestGeneration; + if (!countOnly) { + availableElement.replaceChildren(createMigrationCatalogLoadingState()); + setMigrationCatalogStatus(targetType, "Loading server catalog…"); + } try { - const search = getValue(getMigrationSearchElement(targetType)); - const data = await requestJson(`/api/admin/data-management/migration/catalog/${encodeURIComponent(targetType)}?search=${encodeURIComponent(search)}&limit=50`, { method: "GET" }); - renderMigrationAvailableList(targetType, Array.isArray(data.items) ? data.items : []); + const search = countOnly ? "" : state.search; + const continuationToken = countOnly ? "" : (state.pageTokens[state.pageIndex] || ""); + const pageSize = countOnly ? 1 : 25; + const data = await requestJson( + `/api/admin/data-management/migration/catalog/${encodeURIComponent(targetType)}?search=${encodeURIComponent(search)}&page_size=${pageSize}&continuation_token=${encodeURIComponent(continuationToken)}`, + { method: "GET" } + ); + if (requestGeneration !== state.requestGeneration) { + return; + } + state.totalCount = Number(data.total_count || 0); + if (countOnly) { + updateMigrationScopeCounts(); + return; + } + state.items = Array.isArray(data.items) ? data.items : []; + state.nextToken = data.has_more ? String(data.continuation_token || "") : ""; + setMigrationCatalogStatus( + targetType, + `${formatNumber(state.totalCount)} match${state.totalCount === 1 ? "" : "es"}` + ); + renderMigrationAvailableList(targetType, state.items); + updateMigrationCatalogPagination(targetType); + updateMigrationScopeCounts(); } catch (error) { - availableElement.replaceChildren(createSmallMutedElement(error.message || "Migration catalog could not be loaded.")); + if (requestGeneration !== state.requestGeneration) { + return; + } + const message = error.message || "Migration catalog could not be loaded."; + if (!countOnly) { + availableElement.replaceChildren(createSmallMutedElement(message)); + setMigrationCatalogStatus(targetType, message); + updateMigrationCatalogPagination(targetType); + } } } @@ -1533,11 +1795,15 @@ function renderMigrationAvailableList(targetType, items) { const row = document.createElement("button"); row.type = "button"; row.className = "list-group-item list-group-item-action py-2"; - row.appendChild(createMigrationItemContent(item, "Add")); + const isSelected = migrationSelections[targetType].has(item.id); + row.disabled = isSelected; + row.setAttribute("aria-pressed", String(isSelected)); + row.appendChild(createMigrationItemContent(item, isSelected ? "Selected" : "Add")); row.addEventListener("click", () => { migrationSelections[targetType].set(item.id, item); + renderMigrationAvailableList(targetType, migrationCatalogStates[targetType].items); renderMigrationSelectedList(targetType); - loadMigrationSummary(); + invalidateMigrationReview(); }); availableElement.appendChild(row); }); @@ -1561,11 +1827,18 @@ function renderMigrationSelectedList(targetType) { row.appendChild(createMigrationItemContent(item, "Remove")); row.addEventListener("click", () => { migrationSelections[targetType].delete(item.id); + renderMigrationAvailableList(targetType, migrationCatalogStates[targetType].items); renderMigrationSelectedList(targetType); - loadMigrationSummary(); + invalidateMigrationReview(); }); selectedElement.appendChild(row); }); + const selectedCount = migrationSelections[targetType].size; + setText( + document.querySelector(`[data-migration-selected-count="${targetType}"]`), + formatNumber(selectedCount) + ); + updateMigrationScopeCounts(); } function createMigrationItemContent(item, actionLabel) { @@ -1594,6 +1867,264 @@ function createSmallMutedElement(text) { return element; } +function createIcon(className) { + const icon = document.createElement("i"); + icon.className = className; + icon.setAttribute("aria-hidden", "true"); + return icon; +} + +function createMigrationCatalogLoadingState() { + const wrapper = document.createElement("div"); + wrapper.className = "p-3"; + wrapper.setAttribute("role", "status"); + const label = document.createElement("span"); + label.className = "text-muted small"; + label.textContent = "Loading catalog…"; + wrapper.appendChild(label); + return wrapper; +} + +function setMigrationCatalogStatus(targetType, message) { + setText( + document.querySelector(`[data-migration-catalog-status="${targetType}"]`), + message + ); +} + +function updateMigrationCatalogPagination(targetType) { + const state = migrationCatalogStates[targetType]; + const previousButton = document.querySelector(`[data-migration-page-previous="${targetType}"]`); + const nextButton = document.querySelector(`[data-migration-page-next="${targetType}"]`); + setButtonDisabled(previousButton, state.pageIndex === 0); + setButtonDisabled(nextButton, !state.nextToken); + setText( + document.querySelector(`[data-migration-page-status="${targetType}"]`), + `Page ${formatNumber(state.pageIndex + 1)}` + ); +} + +function updateMigrationScopeCounts() { + let totalCount = 0; + migrationTargetTypes.forEach((targetType) => { + const mode = getValue(getMigrationModeElement(targetType)) || "none"; + const count = mode === "all" + ? migrationCatalogStates[targetType].totalCount + : mode === "selected" + ? migrationSelections[targetType].size + : 0; + totalCount += count; + setText( + document.querySelector(`[data-migration-scope-count="${targetType}"]`), + formatNumber(count) + ); + setText( + document.querySelector(`[data-migration-all-count="${targetType}"]`), + `${formatNumber(count)} ${formatActivityLabel(targetType)} included` + ); + }); + setText( + elements.dataManagementMigrationScopeTotal, + `${formatNumber(totalCount)} principal scope${totalCount === 1 ? "" : "s"} selected` + ); +} + +function showMigrationWorkflowStep(stepIndex, options = {}) { + const normalizedIndex = Math.max(0, Math.min(migrationWorkflowSteps.length - 1, Number(stepIndex))); + if (!options.skipValidation && normalizedIndex > migrationWorkflowState.activeStepIndex) { + if (!validateMigrationWorkflowStep(migrationWorkflowState.activeStepIndex)) { + return; + } + } + migrationWorkflowState.activeStepIndex = normalizedIndex; + migrationWorkflowState.highestVisitedStepIndex = Math.max( + migrationWorkflowState.highestVisitedStepIndex, + normalizedIndex + ); + const activeStep = migrationWorkflowSteps[normalizedIndex]; + document.querySelectorAll("[data-migration-step-panel]").forEach((panel) => { + const isActive = panel.dataset.migrationStepPanel === activeStep; + panel.classList.toggle("d-none", !isActive); + panel.setAttribute("aria-hidden", String(!isActive)); + }); + document.querySelectorAll("[data-migration-step-button]").forEach((button) => { + const buttonIndex = migrationWorkflowSteps.indexOf(button.dataset.migrationStepButton); + const isActive = buttonIndex === normalizedIndex; + button.classList.toggle("is-active", isActive); + button.classList.toggle("is-complete", buttonIndex < normalizedIndex); + if (isActive) { + button.setAttribute("aria-current", "step"); + } else { + button.removeAttribute("aria-current"); + } + }); + clearMigrationStepError(); + updateMigrationWorkflowFooter(); +} + +function updateMigrationWorkflowFooter() { + const activeIndex = migrationWorkflowState.activeStepIndex; + const nextLabels = [ + "Continue to Scope", + "Continue to Content & Options", + "Continue to Review", + "Continue to Confirm", + ]; + setButtonDisabled(elements.dataManagementMigrationBackBtn, activeIndex === 0); + const showNext = activeIndex < migrationWorkflowSteps.indexOf("confirm"); + setElementVisible(elements.dataManagementMigrationNextBtn, showNext); + if (showNext) { + const label = nextLabels[activeIndex] || "Continue"; + elements.dataManagementMigrationNextBtn.replaceChildren( + document.createTextNode(label), + createIcon("bi bi-arrow-right ms-1") + ); + setButtonDisabled( + elements.dataManagementMigrationNextBtn, + activeIndex === migrationWorkflowSteps.indexOf("review") && + !isMigrationReviewCurrentAndReady() + ); + } + setText( + elements.dataManagementMigrationStepPosition, + `Step ${formatNumber(activeIndex + 1)} of ${formatNumber(migrationWorkflowSteps.length)}` + ); +} + +function advanceMigrationWorkflow() { + const currentIndex = migrationWorkflowState.activeStepIndex; + if (!validateMigrationWorkflowStep(currentIndex)) { + return; + } + showMigrationWorkflowStep(currentIndex + 1, { skipValidation: true }); +} + +function validateMigrationWorkflowStep(stepIndex) { + const step = migrationWorkflowSteps[stepIndex]; + clearMigrationStepError(); + if (step === "target" && !getValue(elements.datamanagementtargetcosmosendpoint)) { + showMigrationStepError("Enter the target Cosmos endpoint before choosing migration scope.", "target"); + elements.datamanagementtargetcosmosendpoint?.focus(); + return false; + } + if (step === "scope") { + const selectedCount = migrationTargetTypes.reduce((total, targetType) => { + const mode = getValue(getMigrationModeElement(targetType)) || "none"; + if (mode === "all") { + return total + migrationCatalogStates[targetType].totalCount; + } + if (mode === "selected") { + return total + migrationSelections[targetType].size; + } + return total; + }, 0); + if (selectedCount === 0) { + showMigrationStepError("Choose at least one user, group, or public workspace before continuing.", "scope"); + return false; + } + } + if ( + step === "options" && + elements.datamanagementmigrationincludeaisearch?.checked && + !elements.datamanagementmigrationtargetsearchwritesfrozen?.checked + ) { + showMigrationStepError("Confirm that external destination AI Search writers are frozen before review.", "options"); + elements.datamanagementmigrationtargetsearchwritesfrozen?.focus(); + return false; + } + if (step === "review" && !isMigrationReviewCurrentAndReady()) { + showMigrationStepError("Run a current preflight review and resolve every blocker before confirmation.", "review"); + return false; + } + return true; +} + +function showMigrationStepError(message, workflowStep = "") { + const errorElement = elements.dataManagementMigrationStepError; + if (!errorElement) { + return; + } + setText(errorElement, message); + errorElement.classList.remove("d-none"); + errorElement.focus(); + document.querySelectorAll("[data-migration-step-button]").forEach((button) => { + button.classList.toggle( + "has-error", + Boolean(workflowStep) && button.dataset.migrationStepButton === workflowStep + ); + }); +} + +function clearMigrationStepError() { + setElementVisible(elements.dataManagementMigrationStepError, false); + document.querySelectorAll("[data-migration-step-button]").forEach((button) => { + button.classList.remove("has-error"); + }); +} + +function invalidateMigrationReview() { + migrationWorkflowState.reviewRequestGeneration += 1; + if (migrationWorkflowState.review && !migrationWorkflowState.reviewStale) { + migrationWorkflowState.reviewStale = true; + showMigrationStepError( + "Migration inputs changed. Run preflight review again before confirmation.", + "review" + ); + } + if (elements.datamanagementmigrationfinalconfirmation) { + elements.datamanagementmigrationfinalconfirmation.checked = false; + } + updateMigrationReadiness(); + updateMigrationExecutionButtonState(); + updateMigrationWorkflowFooter(); +} + +function isMigrationReviewCurrentAndReady() { + return Boolean( + migrationWorkflowState.review?.ready && + !migrationWorkflowState.reviewStale + ); +} + +function updateMigrationReadiness() { + const readiness = elements.dataManagementMigrationReadiness; + if (!readiness) { + return; + } + readiness.classList.remove("is-ready", "is-blocked", "is-stale"); + let message = "Not reviewed"; + if (migrationWorkflowState.reviewStale && migrationWorkflowState.review) { + readiness.classList.add("is-stale"); + message = "Review is stale"; + } else if (migrationWorkflowState.review?.ready) { + readiness.classList.add("is-ready"); + message = migrationWorkflowState.review.warning_count + ? `Ready with ${formatNumber(migrationWorkflowState.review.warning_count)} warning${migrationWorkflowState.review.warning_count === 1 ? "" : "s"}` + : "Ready to confirm"; + } else if (migrationWorkflowState.review) { + readiness.classList.add("is-blocked"); + message = `${formatNumber(migrationWorkflowState.review.blocker_count || 0)} blocker${migrationWorkflowState.review.blocker_count === 1 ? "" : "s"}`; + } + const label = readiness.querySelector("span:last-child"); + setText(label, message); +} + +function updateMigrationExecutionButtonState() { + const isMirror = getMigrationTransferMode() === "mirror_with_deletions"; + const mirrorConfirmed = ( + !isMirror || + getValue(elements.datamanagementmigrationmirrorconfirmationphrase) === migrationMirrorConfirmationPhrase + ); + const canExecute = ( + isMigrationReviewCurrentAndReady() && + Boolean(elements.datamanagementmigrationfinalconfirmation?.checked) && + mirrorConfirmed && + !migrationWorkflowState.submissionInFlight && + !migrationWorkflowState.submissionAccepted + ); + setButtonDisabled(elements.dataManagementExecuteMigrationBtn, !canExecute); +} + function buildMigrationPlan() { const migrationMode = getMigrationTransferMode(); return { @@ -1635,6 +2166,7 @@ function updateMigrationModeVisibility() { if (elements.dataManagementMigrationModeDescription) { elements.dataManagementMigrationModeDescription.textContent = descriptions[migrationMode] || descriptions.new_only; } + updateMigrationExecutionButtonState(); } function buildMigrationSelection(targetType) { @@ -1646,65 +2178,101 @@ function buildMigrationSelection(targetType) { }; } -async function loadMigrationSummary(triggerButton = null, showSuccess = false) { +async function loadMigrationSummary(triggerButton = null) { + return runMigrationReview(triggerButton); +} + +async function runMigrationReview(triggerButton = null) { const summaryElement = elements.dataManagementMigrationSummary; if (!summaryElement) { return; } - setBusy(triggerButton, true, "Previewing..."); + clearMigrationStepError(); + const requestGeneration = migrationWorkflowState.reviewRequestGeneration + 1; + migrationWorkflowState.reviewRequestGeneration = requestGeneration; + setBusy(triggerButton, true, "Running preflight…"); + setElementVisible(elements.dataManagementMigrationReviewEmpty, false); + setElementVisible(summaryElement, true); + summaryElement.replaceChildren(createMigrationCatalogLoadingState()); try { - const data = await requestJson("/api/admin/data-management/migration/summary", { + const data = await requestJson("/api/admin/data-management/migration/review", { method: "POST", body: JSON.stringify({ + settings: collectSettings(), migration_plan: buildMigrationPlan(), - include_inventory: Boolean(showSuccess), }), }); - renderMigrationSummary(data.summary || {}, data.preview || null); - if (showSuccess) { - setStatus("Migration preview refreshed.", "success"); - showToast("Migration preview refreshed.", "success"); + if (requestGeneration !== migrationWorkflowState.reviewRequestGeneration) { + return; } + migrationWorkflowState.review = data.review || null; + migrationWorkflowState.reviewStale = false; + renderMigrationReview(migrationWorkflowState.review || {}); + updateMigrationReadiness(); + updateMigrationExecutionButtonState(); + updateMigrationWorkflowFooter(); + const isReady = migrationWorkflowState.review?.ready === true; + const message = isReady + ? "Migration preflight review is ready for confirmation." + : "Migration preflight review found blockers."; + setStatus(message, isReady ? "success" : "warning"); + showToast(message, isReady ? "success" : "warning"); } catch (error) { - summaryElement.replaceChildren(createSmallMutedElement(error.message || "Migration summary could not be loaded.")); - if (showSuccess) { - setStatus(error.message || "Migration summary could not be loaded.", "danger"); - showToast(error.message || "Migration summary could not be loaded.", "danger"); + if (requestGeneration !== migrationWorkflowState.reviewRequestGeneration) { + return; } + migrationWorkflowState.reviewStale = true; + summaryElement.replaceChildren(createSmallMutedElement(error.message || "Migration summary could not be loaded.")); + showMigrationStepError( + error.message || "Migration review could not be completed.", + error.workflowStep || "review" + ); + setStatus(error.message || "Migration review could not be completed.", "danger"); + showToast(error.message || "Migration review could not be completed.", "danger"); + updateMigrationReadiness(); + updateMigrationExecutionButtonState(); } finally { setBusy(triggerButton, false); } } +function renderMigrationReview(review) { + setElementVisible(elements.dataManagementMigrationReviewEmpty, false); + setElementVisible(elements.dataManagementMigrationSummary, true); + setElementVisible(elements.dataManagementMigrationReviewChecks, true); + setElementVisible(elements.dataManagementRefreshMigrationSummaryBtn, true); + renderMigrationSummary(review.summary || {}, review.preview || null); + renderMigrationReviewChecks(Array.isArray(review.checks) ? review.checks : []); + renderMigrationConfirmSummary(review); +} + function renderMigrationSummary(summary, preview = null) { const summaryElement = elements.dataManagementMigrationSummary; if (!summaryElement) { return; } const container = document.createElement("div"); - container.className = "row g-3"; + container.className = "migration-review-grid"; migrationTargetTypes.forEach((targetType) => { const targetSummary = summary[targetType] || {}; container.appendChild(createMigrationSummaryCard(targetType, targetSummary)); }); - const optionsColumn = document.createElement("div"); - optionsColumn.className = "col-12"; const options = document.createElement("div"); - options.className = "d-flex flex-wrap gap-2"; + options.className = "d-flex flex-wrap gap-2 mt-3"; options.appendChild(createBadge(summary.include_ai_search ? "AI Search included" : "AI Search skipped", summary.include_ai_search ? "bg-info text-dark" : "bg-secondary")); options.appendChild(createBadge(summary.include_source_blobs ? "Source blobs included" : "Source blobs skipped", summary.include_source_blobs ? "bg-info text-dark" : "bg-secondary")); options.appendChild(createBadge(`Mode: ${formatActivityLabel(summary.migration_mode || "new_only")}`, summary.migration_mode === "mirror_with_deletions" ? "bg-danger" : "bg-primary")); - optionsColumn.appendChild(options); - container.appendChild(optionsColumn); + const wrapper = document.createElement("div"); + wrapper.append(container, options); if (preview?.estimated_outcomes) { - container.appendChild(createMigrationPreviewOutcomes(preview)); + wrapper.appendChild(createMigrationPreviewOutcomes(preview)); } - summaryElement.replaceChildren(container); + summaryElement.replaceChildren(wrapper); } function createMigrationPreviewOutcomes(preview) { const column = document.createElement("div"); - column.className = "col-12 border-top pt-3"; + column.className = "border-top mt-3 pt-3"; const heading = document.createElement("div"); heading.className = "fw-semibold mb-2"; heading.textContent = "Estimated destination changes"; @@ -1730,31 +2298,101 @@ function createMigrationPreviewOutcomes(preview) { } function createMigrationSummaryCard(targetType, targetSummary) { - const column = document.createElement("div"); - column.className = "col-md-4"; const wrapper = document.createElement("div"); - wrapper.className = "border rounded p-3 h-100"; + wrapper.className = "migration-review-metric"; const title = document.createElement("div"); - title.className = "fw-semibold"; + title.className = "text-muted small"; title.textContent = formatActivityLabel(targetType); const count = document.createElement("div"); - count.className = "fs-5 fw-semibold"; - count.textContent = formatNumber(targetSummary.count || 0); + count.className = "fw-semibold fs-5"; + count.textContent = `${formatNumber(targetSummary.count || 0)} principal${Number(targetSummary.count || 0) === 1 ? "" : "s"}`; const mode = document.createElement("div"); mode.className = "text-muted small"; - mode.textContent = `${formatActivityLabel(targetSummary.mode || "none")} - ${targetSummary.include_documents ? "documents included" : "documents skipped"}`; - const details = document.createElement("details"); - details.className = "mt-2"; - const summaryLabel = document.createElement("summary"); - summaryLabel.className = "small"; - summaryLabel.textContent = "View selected IDs"; - const idList = document.createElement("div"); - idList.className = "small text-muted text-break mt-1"; - idList.textContent = Array.isArray(targetSummary.ids) && targetSummary.ids.length ? targetSummary.ids.join(", ") : "No explicit selected IDs."; - details.append(summaryLabel, idList); - wrapper.append(title, count, mode, details); - column.appendChild(wrapper); - return column; + mode.textContent = targetSummary.include_documents + ? `${formatActivityLabel(targetSummary.mode || "none")} · ${formatNumber(targetSummary.document_count || 0)} documents` + : `${formatActivityLabel(targetSummary.mode || "none")} · documents skipped`; + wrapper.append(title, count, mode); + return wrapper; +} + +function renderMigrationReviewChecks(checks) { + const container = elements.dataManagementMigrationReviewChecks; + if (!container) { + return; + } + container.className = "migration-review-checks"; + container.replaceChildren(); + checks.forEach((check) => { + const status = ["pass", "warning", "block"].includes(check.status) ? check.status : "block"; + const row = document.createElement("div"); + row.className = `migration-review-check is-${status}`; + const iconClasses = { + pass: "bi bi-check-circle-fill", + warning: "bi bi-exclamation-triangle-fill", + block: "bi bi-x-octagon-fill", + }; + const icon = createIcon(iconClasses[status]); + const copy = document.createElement("div"); + const title = document.createElement("strong"); + title.textContent = check.label || "Review check"; + const summary = document.createElement("p"); + summary.textContent = check.summary || "No details were returned."; + copy.append(title, summary); + const badgeClasses = { + pass: "bg-success", + warning: "bg-warning text-dark", + block: "bg-danger", + }; + row.append(icon, copy, createBadge(formatActivityLabel(status), badgeClasses[status])); + container.appendChild(row); + }); +} + +function renderMigrationConfirmSummary(review) { + const container = elements.dataManagementMigrationConfirmSummary; + if (!container) { + return; + } + const summary = review.summary || {}; + const preview = review.preview || {}; + const outcomes = preview.estimated_outcomes || {}; + const principalCount = migrationTargetTypes.reduce( + (total, targetType) => total + Number(summary[targetType]?.count || 0), + 0 + ); + const documentCount = migrationTargetTypes.reduce( + (total, targetType) => total + Number(summary[targetType]?.document_count || 0), + 0 + ); + const heading = document.createElement("div"); + heading.className = "d-flex flex-wrap justify-content-between align-items-center gap-2 mb-3"; + const title = document.createElement("strong"); + title.textContent = "Server-normalized execution plan"; + const readinessBadge = createBadge( + review.ready ? "Ready" : "Blocked", + review.ready ? "bg-success" : "bg-danger" + ); + heading.append(title, readinessBadge); + const metrics = document.createElement("div"); + metrics.className = "migration-review-grid"; + [ + ["Principal scopes", formatNumber(principalCount)], + ["Included documents", formatNumber(documentCount)], + ["Synchronization mode", formatActivityLabel(summary.migration_mode || "new_only")], + ["Creates / updates", `${formatNumber(outcomes.create_count || 0)} / ${formatNumber(outcomes.update_count || 0)}`], + ["Deletes", formatNumber(outcomes.delete_count || 0)], + ["Conflicts", formatNumber(outcomes.conflict_count || 0)], + ].forEach(([label, value]) => { + const metric = document.createElement("div"); + metric.className = "migration-review-metric"; + const metricLabel = document.createElement("span"); + metricLabel.textContent = label; + const metricValue = document.createElement("strong"); + metricValue.textContent = value; + metric.append(metricLabel, metricValue); + metrics.appendChild(metric); + }); + container.replaceChildren(heading, metrics); } function formatOperation(operation, backupType) { @@ -1764,6 +2402,236 @@ function formatOperation(operation, backupType) { return operation.replace(/_/g, " "); } +function attachMigrationWorkflowJob(job) { + if (!job?.id || job.operation !== "migration") { + return; + } + migrationWorkflowState.currentJob = job; + setElementVisible(elements.dataManagementMigrationProgressEmpty, false); + setElementVisible(elements.dataManagementMigrationWorkflowProgress, true); + setElementVisible(elements.dataManagementMigrationViewJobBtn, true); + setElementVisible( + elements.dataManagementMigrationStartNewBtn, + !activeJobStatuses.has(String(job.status || "")) + ); + renderMigrationWorkflowProgress(job); + if (activeJobStatuses.has(String(job.status || ""))) { + startMigrationWorkflowRefresh(); + } else { + stopMigrationWorkflowRefresh(); + } +} + +function resetMigrationWorkflowForNewRun() { + const currentStatus = String(migrationWorkflowState.currentJob?.status || ""); + if (currentStatus && activeJobStatuses.has(currentStatus)) { + return; + } + stopMigrationWorkflowRefresh(); + migrationWorkflowState.activeStepIndex = 0; + migrationWorkflowState.highestVisitedStepIndex = 0; + migrationWorkflowState.review = null; + migrationWorkflowState.reviewStale = true; + migrationWorkflowState.reviewRequestGeneration += 1; + migrationWorkflowState.submissionInFlight = false; + migrationWorkflowState.submissionAccepted = false; + migrationWorkflowState.currentJob = null; + setChecked(elements.datamanagementmigrationfinalconfirmation, false); + setValue(elements.datamanagementmigrationmirrorconfirmationphrase, ""); + setElementVisible(elements.dataManagementMigrationProgressEmpty, true); + setElementVisible(elements.dataManagementMigrationWorkflowProgress, false); + setElementVisible(elements.dataManagementMigrationViewJobBtn, false); + setElementVisible(elements.dataManagementMigrationStartNewBtn, false); + setElementVisible(elements.dataManagementMigrationReviewEmpty, true); + setElementVisible(elements.dataManagementMigrationSummary, false); + setElementVisible(elements.dataManagementMigrationReviewChecks, false); + setElementVisible(elements.dataManagementRefreshMigrationSummaryBtn, false); + updateMigrationReadiness(); + updateMigrationExecutionButtonState(); + showMigrationWorkflowStep(0, { skipValidation: true }); +} + +function renderMigrationWorkflowProgress(job) { + renderMigrationWorkflowProgressSummary(job); + renderMigrationWorkflowProgressState(job); + renderMigrationWorkflowProgressActions(job); +} + +function renderMigrationWorkflowProgressSummary(job) { + const container = elements.dataManagementMigrationWorkflowProgressSummary; + if (!container) { + return; + } + const summary = document.createElement("div"); + summary.className = "migration-review-grid mb-3"; + [ + ["Status", formatStatusLabel(job.status || "unknown")], + ["Job ID", job.id || "N/A"], + ["Created", formatDate(job.created_at)], + ].forEach(([label, value]) => { + const metric = document.createElement("div"); + metric.className = "migration-review-metric"; + const metricLabel = document.createElement("span"); + metricLabel.textContent = label; + const metricValue = document.createElement("strong"); + metricValue.className = "text-break"; + metricValue.textContent = value; + metric.append(metricLabel, metricValue); + summary.appendChild(metric); + }); + container.replaceChildren(summary); +} + +function renderMigrationWorkflowProgressState(job) { + const container = elements.dataManagementMigrationWorkflowProgressState; + if (!container) { + return; + } + const progress = job.progress && typeof job.progress === "object" ? job.progress : {}; + const percent = getProgressPercent(progress); + const active = activeJobStatuses.has(String(job.status || "")); + const panel = document.createElement("div"); + panel.className = "migration-confirm-summary"; + const header = document.createElement("div"); + header.className = "d-flex flex-wrap justify-content-between align-items-center gap-2 mb-2"; + const title = document.createElement("strong"); + title.textContent = progress.current_step + ? formatActivityLabel(progress.current_step) + : "Waiting for migration progress"; + const status = document.createElement("span"); + status.className = "small text-muted"; + status.textContent = active + ? "Live updates every 2 seconds" + : `Job is ${formatStatusLabel(job.status || "unknown")}`; + header.append(title, status); + + const progressWrapper = document.createElement("div"); + progressWrapper.className = "progress"; + progressWrapper.setAttribute("role", "progressbar"); + progressWrapper.setAttribute("aria-valuemin", "0"); + progressWrapper.setAttribute("aria-valuemax", "100"); + const progressBar = document.createElement("div"); + if (active) { + progressWrapper.setAttribute("aria-valuetext", "Migration stage is active; measured throughput is shown below."); + progressBar.className = "progress-bar progress-bar-striped progress-bar-animated"; + progressBar.style.width = "100%"; + progressBar.textContent = "Active stage"; + } else { + progressWrapper.setAttribute("aria-valuenow", String(percent)); + progressBar.className = `progress-bar ${job.status === "failed" || job.status === "canceled" ? "bg-danger" : "bg-success"}`; + progressBar.style.width = `${percent}%`; + progressBar.textContent = `${percent}%`; + } + progressWrapper.appendChild(progressBar); + const message = document.createElement("p"); + message.className = "small text-muted mt-2 mb-0"; + message.textContent = job.last_message || job.last_error || "No job message has been recorded yet."; + panel.append(header, progressWrapper, message); + + const metrics = getMigrationLiveMetrics(job); + if (metrics.length) { + const metricGrid = document.createElement("div"); + metricGrid.className = "migration-review-grid mt-3"; + metrics.forEach((metric) => { + const item = document.createElement("div"); + item.className = "migration-review-metric"; + const label = document.createElement("span"); + label.textContent = metric.label; + const value = document.createElement("strong"); + value.textContent = metric.value; + item.append(label, value); + metricGrid.appendChild(item); + }); + panel.appendChild(metricGrid); + } + container.replaceChildren(panel); +} + +function renderMigrationWorkflowProgressActions(job) { + const container = elements.dataManagementMigrationWorkflowProgressActions; + if (!container) { + return; + } + container.replaceChildren(); + if (job.can_retry === true) { + const retryButton = document.createElement("button"); + retryButton.type = "button"; + retryButton.className = "btn btn-outline-warning"; + retryButton.append( + createIcon("bi bi-arrow-repeat me-1"), + document.createTextNode(job.status === "running" ? "Resume" : "Retry") + ); + retryButton.addEventListener("click", () => retryDataManagementJob(job, retryButton)); + container.appendChild(retryButton); + } + if (job.can_cancel === true) { + const cancelButton = document.createElement("button"); + cancelButton.type = "button"; + cancelButton.className = "btn btn-outline-danger"; + cancelButton.append( + createIcon("bi bi-stop-circle me-1"), + document.createTextNode("Cancel") + ); + cancelButton.addEventListener("click", () => openDataManagementCancellationModal(job)); + container.appendChild(cancelButton); + } + const logButton = document.createElement("button"); + logButton.type = "button"; + logButton.className = "btn btn-outline-primary"; + logButton.append( + createIcon("bi bi-list-check me-1"), + document.createTextNode("View warnings and item log") + ); + logButton.addEventListener("click", () => loadDataManagementJobDetail(job.id)); + container.appendChild(logButton); +} + +function startMigrationWorkflowRefresh() { + if (migrationWorkflowRefreshTimer || !migrationWorkflowState.currentJob?.id) { + return; + } + migrationWorkflowRefreshTimer = window.setInterval( + refreshMigrationWorkflowProgress, + jobDetailRefreshIntervalMs + ); +} + +async function refreshMigrationWorkflowProgress() { + const jobId = migrationWorkflowState.currentJob?.id; + if (!jobId || migrationWorkflowRefreshInFlight) { + return; + } + migrationWorkflowRefreshInFlight = true; + try { + const data = await requestJson( + `/api/admin/data-management/jobs/${encodeURIComponent(jobId)}/progress`, + { method: "GET" } + ); + const job = data.job || {}; + attachMigrationWorkflowJob(job); + if (!activeJobStatuses.has(String(job.status || ""))) { + stopMigrationWorkflowRefresh(); + loadDataManagementJobs(); + loadDataManagementBackups(); + } + } catch (error) { + stopMigrationWorkflowRefresh(); + showMigrationStepError( + "Live migration updates paused after a request failure. Open the full job log to refresh.", + "progress" + ); + } finally { + migrationWorkflowRefreshInFlight = false; + } +} + +function stopMigrationWorkflowRefresh() { + if (migrationWorkflowRefreshTimer) { + window.clearInterval(migrationWorkflowRefreshTimer); + migrationWorkflowRefreshTimer = null; + } +} + function createHistoryListState() { return { currentToken: null, @@ -2252,6 +3120,7 @@ async function requestDataManagementCancellation() { ); window.bootstrap.Modal.getOrCreateInstance(elements.dataManagementMigrationCancelModal).hide(); renderJobs([data.job]); + attachMigrationWorkflowJob(data.job); loadDataManagementJobs(); if (currentJobDetailId === jobId) { loadDataManagementJobDetail(jobId, { showModal: false, liveRefresh: true }); @@ -2278,6 +3147,7 @@ async function retryDataManagementJob(job, button) { setStatus(`${operationLabel} retry queued from durable checkpoints.`, "success"); showToast(`${operationLabel} retry queued.`, "success"); renderJobs([data.job]); + attachMigrationWorkflowJob(data.job); loadDataManagementJobs(); loadDataManagementJobDetail(jobId, { showModal: false, liveRefresh: true }); } catch (error) { diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index 8cfa1e9b8..9bd8fb62a 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -5810,299 +5810,583 @@
Backup Operations
-
-
-
- -
+ +
+
-

Migration

-

Use migration when moving SimpleChat data into another SimpleChat environment, rehearsing a cutover, or preparing a controlled environment transfer.

-
-
-

- Migration targets a separate Cosmos account that will host the standard SimpleChat database and container layout. The database name is fixed to SimpleChat so exported metadata, container mappings, and future apply jobs use the same app contract in every environment. -

- -
-
-
-
Target Cosmos Database
-

Fill out the target Cosmos account connection that migration jobs will validate and write to.

-
-
+
+ + Not reviewed +
+ + +
+
-
- For managed identity, assign this App Service identity Cosmos DB Data Contributor on the target account and allow network access from this application environment. -
-
-
- - -
-
- - -
-
- - -
Always SimpleChat.
-
-
-
- - -
-
-
- - -
-
- - -
-
-
- -
-
-
-
Target Search
-

Migration copies selected indexed documents into matching SimpleChat AI Search indexes on the target search service.

-
- -
-
-
- - -
-
- - -
-
- - -
-
-
- -
-
-
-
Target Enhanced Citation Storage
-

Required only when migrating source document blobs for Enhanced Citations.

-
- -
-
-
- - -
-
- - -
-
- - + + + + + + + + +
+
+
+
Connect the destination
+

Configure the services this migration will write to. Stored credentials remain redacted.

+
+ Destination database: SimpleChat
-
-
-
-
-
-
Migration Workflow
-

Choose users, groups, and public workspaces to migrate, then choose whether to include each scope's documents, AI Search entries, and source blobs.

+
+
+
+
+
Target Cosmos Database
+

Required for every migration.

+
+ +
+
+ Managed identity requires Cosmos DB Data Contributor and target network access. +
+
+
+ + +
+
+ + +
+
+ + +
Fixed app contract.
+
+
+
+ + +
+
+
+ + +
+
+ + +
+
+
+ +
+
+
+
Target Search
+

Required when AI Search documents are included.

+
+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+
+
+
Target Enhanced Citation Storage
+

Required only when source document blobs are included.

+
+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+
- -
-
- Synchronization mode -
- - - - - - -
-

- Copies source items that are absent from the destination. Existing destination data is never updated or deleted. -

-
-
- - -
The baseline fixes the prior service watermarks used for this catch-up run.
+ + + +
+
+
+
Choose who and what moves
+

Selections persist while you search and page. “All” always uses the exhaustive server count.

+
0 principal scopes selected
-
-
-
-
-
Users
- -
- - -
-
Available
-
-
Selected
-
-
- - + +
+ +
+ Migration mode +
+ + + +
+
+ +
+
+
+
+
Available users
+

Search the server catalog.

+
+
+ + + +
+
+
+
+ + Page 1 + +
+
-
-
-
-
Groups
- -
- - -
-
Available
-
-
Selected
-
-
- - + +
+ +
+ Loading exhaustive count… + Every current user record will be resolved by the server when the job starts.
-
-
-
-
Public Workspaces
- -
- - -
-
Available
-
-
Selected
-
-
- - + +
+ + +
+ + +
+ +
+ Migration mode +
+ + + +
+
+
+
+
+
+
Available groups
+

Search the server catalog.

+
+
+ + + +
+
+
+
+ + Page 1 + +
+
+ +
+
+ +
+ Loading exhaustive count… + Every current group record will be resolved by the server when the job starts. +
+
+
+ + +
+
+ +
+ +
+ Migration mode +
+ + + +
+
+
+
+
+
+
Available public workspaces
+

Search the server catalog.

+
+
+ + + +
+
+
+
+ + Page 1 + +
+
+ +
+
+ +
+ Loading exhaustive count… + Every current public workspace record will be resolved by the server when the job starts.
+
+ + +
+
+ + + +
+
+
+
Choose transfer behavior
+

Set synchronization semantics, included services, retry behavior, and optional destination capacity.

+
-
-
-
-
- - + +
+ Synchronization mode +
+ + + + + + +
+

+ Copies source items that are absent from the destination. Existing destination data is never updated or deleted. +

+
+
+ + +
The baseline fixes the prior service watermarks used for this catch-up run.
+
+
+
+ +
+ Data surfaces +
+
+
+ + +
+
+ + +
SimpleChat pauses its own target indexing. Freeze other writers before review.
+
+
+
+
+ + +
Requires Enhanced Citation storage at both source and destination.
+
+
-
- - -
Azure AI Search does not support conditional document writes. SimpleChat target indexing pauses automatically; freeze all external writers before copying Search documents.
+
+ +
+ Performance and resume +

Migration uses durable resource checkpoints and retains the same migration ID after Retry or Resume.

+
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+
Eligible capacity is raised only to 10,000 RU/s and restored after completion or failure. ARM permission is required.
+
+
+
+ + + +
+
+
+
Prove the plan is ready
+

Preflight runs server-owned access probes and inventory. Any earlier change makes this review stale.

+
+
+ +
-
-
- - +
+ +
+ Review has not run. + Run preflight to verify target access, counts, collisions, locks, and capacity policy.
-
-
-
+
+
+ +
+ + +
+
-
Performance And Resume
-

Migration uses durable per-resource checkpoints and the same migration ID after a retry.

+
Confirm execution
+

Review the final server-normalized plan. Submission is guarded against duplicate requests.

-
-
-
- - +
+

Complete preflight review before confirmation.

+
+ +
+ + +
+ +
+ + +
+
+
+
Operate the migration
+

Progress comes from the durable job record. Cancel, Retry, and Resume retain verified checkpoints.

-
- - +
+ +
-
- - +
+
+ +
+ No migration is attached to this workflow yet. + After execution, this stage follows the queued job and exposes its recovery actions.
-
- - +
+
+
+
-
-
- - -
-
-
The migration records the prior database or dedicated-container capacity, raises only eligible targets up to 10,000 RU/s, and restores it after completion or failure. Capacity changes require destination ARM permission.
-
+
+ +
+
+ +
-
-
-

Choose migration targets to see the summary.

-
-
- - -
+ Step 1 of 6 +
-
+
diff --git a/docs/explanation/features/DATA_MANAGEMENT_BACKUP_MIGRATION.md b/docs/explanation/features/DATA_MANAGEMENT_BACKUP_MIGRATION.md index 9615b32c2..b342e1ee9 100644 --- a/docs/explanation/features/DATA_MANAGEMENT_BACKUP_MIGRATION.md +++ b/docs/explanation/features/DATA_MANAGEMENT_BACKUP_MIGRATION.md @@ -1,7 +1,7 @@ # Data Management Backup and Migration Implemented in version: **0.241.211** -Updated in version: **0.250.104** +Updated in version: **0.250.105** ## Overview @@ -95,16 +95,18 @@ Partial backup details use the same artifact layout but expose immutable cutoff ### Migration Workflow -The Migration card supports a guided migration workflow: +The Migration card uses a six-stage administrator workflow: -- Configure Target Cosmos DB, Target Search, and Target Enhanced Citation Storage, with test buttons for each target service. -- Select whether to migrate no users, all users, or selected users, with optional user document migration. -- Select whether to migrate no groups, all groups, or selected groups, with optional group document migration. -- Select whether to migrate no public workspaces, all public workspaces, or selected public workspaces, with optional public workspace document migration. -- Preview the migration plan to refresh counts and selected IDs before execution. -- Choose New only, Delta / upsert, or explicitly confirmed Mirror with deletions. Delta and mirror pin a compatible completed migration as their prior watermark. -- Run a read-only live inventory preview for estimated create, update, unchanged, delete, missing, not-applicable, and conflict counts. Execution also captures its own durable inventory after the worker acquires the global migration lock. -- Execute Migration queues a durable Data Management migration job. The job history modal shows live progress, per-step timeline events, migrated artifact counts, and warnings. +1. **Target** configures Target Cosmos DB, Target Search, and Target Enhanced Citation Storage without returning stored secrets to the browser. +2. **Scope** selects `none`, `selected`, or `all` for users, groups, and public workspaces. Selected mode keeps choices while searching or paging. All mode displays an exhaustive server-owned count rather than treating one catalog page as the full population. +3. **Content & Options** chooses included documents, AI Search entries, source blobs, New only, Delta / upsert, or Mirror with deletions, plus retry/concurrency and optional destination capacity. +4. **Review** runs a server-owned preflight snapshot. It reports authoritative principal/document counts, included and skipped surfaces, destination inventory changes, collisions, partition-key compatibility, source and target access, capacity policy, destination coordinator/write-gate readiness, and blockers or warnings. +5. **Confirm** presents the normalized plan and requires a separate acknowledgement. Mirror mode also requires the exact `MIRROR WITH DELETIONS` phrase. +6. **Progress** follows the sanitized durable job record and exposes progress, throughput, failures, collisions, warnings, Cancel, Retry/Resume, and the full job log. + +Catalog APIs use bounded page sizes, deterministic ID ordering, server-side search, authoritative filtered totals, and opaque continuation state bound to the current scope type and search. Invalid or mismatched continuation input fails safely. Explicit selected scope is bounded to 2,000 IDs and fails validation instead of silently truncating. + +Any target, scope, or option change marks the review stale and prevents confirmation until preflight runs again. Ready reviews issue a short-lived, administrator-bound authorization that is atomically reserved before durable job creation and consumed by that exact job before work starts, so blocked, expired, replayed, or changed reviews cannot queue work. A failed job create releases the exact reservation for retry. Submission also has an explicit in-flight guard, and API validation errors return to the relevant workflow stage. The worker rejects execution if migration-relevant settings changed after queueing, then captures its own durable inventory and reruns authoritative preflight after it acquires the migration lease and destination coordinator. A terminal Progress view exposes **Start new migration** to deliberately reset the consumed confirmation. Migration execution currently copies selected SimpleChat Cosmos records, matching AI Search documents for selected document scopes, and source document blobs when Enhanced Citations source and destination storage are configured. @@ -155,12 +157,13 @@ Data Management settings save through their own API and are excluded from the re 5. Configure the full backup cadence and scheduled UTC time. 6. Configure Cosmos Backup Performance when needed. Keep the default bounded concurrency for normal workloads; enable the source boost only when the cost, topology, and ARM permissions have been reviewed. 7. Queue a full or partial backup, or use the restore/migration dry-run buttons to create durable orchestration records. -8. Configure and test Target Cosmos, Target Search, and Target Enhanced Citation Storage before running an actual migration. -9. Use the Migration Workflow to choose users, groups, and public workspaces, then decide whether documents, AI Search entries, and source blobs should be included. -10. Choose the synchronization mode, preview live destination changes, and then Execute Migration to queue the job. Mirror mode requires the exact destructive confirmation phrase. -11. Open Advanced backup scope only when you need to alter the default Cosmos DB, AI Search index, or source blob backup surfaces. -12. Use Backup Inventory to see available backups first, filter to full or partial backups, and open View Log for structured backup details. -13. Use Job History to inspect active container, checkpoint position, records, bytes, RU, retries/throttles, rates, capacity restoration state, durable backup cutoff/checkpoint state, completed steps, reconciliation readiness, preview divergence, and artifact contents. Backup and migration detail both provide Retry/Resume and Cancel; migration detail also provides full or failure-only JSONL manifest downloads. +8. In Migration, complete Target, Scope, and Content & Options. Use server-paginated search for selected principals or choose All to use the exhaustive count. +9. Run Preflight Review and resolve every blocker. Re-run it after changing any earlier-stage input. +10. Continue to Confirm, acknowledge the normalized plan, and provide the exact destructive phrase for Mirror with deletions. +11. Execute once, then use Progress or the full job log for live telemetry, Cancel, Retry/Resume, warnings, and remediation details. +12. Open Advanced backup scope only when you need to alter the default Cosmos DB, AI Search index, or source blob backup surfaces. +13. Use Backup Inventory to see available backups first, filter to full or partial backups, and open View Log for structured backup details. +14. Use Job History to inspect active container, checkpoint position, records, bytes, RU, retries/throttles, rates, capacity restoration state, durable backup cutoff/checkpoint state, completed steps, reconciliation readiness, preview divergence, and artifact contents. Backup and migration detail both provide Retry/Resume and Cancel; migration detail also provides full or failure-only JSONL manifest downloads. Migration is used when moving SimpleChat data into another SimpleChat environment, rehearsing a cutover, or preparing a controlled environment transfer. The target Cosmos account, authentication type, and optional account key are configurable. The target database name is fixed to `SimpleChat` so future migration apply jobs use the standard SimpleChat container layout. @@ -171,6 +174,7 @@ For optional local/source Cosmos backup capacity management, assign the App Serv ## Testing and Validation - Functional security coverage: `functional_tests/test_data_management_security_patterns.py`. +- Migration workflow contract coverage: `functional_tests/test_data_management_migration_workflow_contract.py`. - History pagination, filtering, deterministic ties, continuation validation, global summary, and sanitization coverage: `functional_tests/test_data_management_history_pagination.py`. - Cosmos composite-index maintenance coverage: `functional_tests/test_cosmos_wave3a_indexing_maintenance.py`. - Authenticated pagination, filter reset, refresh preservation, responsive controls, and request-order guard coverage: `ui_tests/test_admin_data_management_settings_ui.py`. diff --git a/docs/explanation/features/DATA_MANAGEMENT_MIGRATION_RESILIENCE.md b/docs/explanation/features/DATA_MANAGEMENT_MIGRATION_RESILIENCE.md index 68757740c..5bfc5b12c 100644 --- a/docs/explanation/features/DATA_MANAGEMENT_MIGRATION_RESILIENCE.md +++ b/docs/explanation/features/DATA_MANAGEMENT_MIGRATION_RESILIENCE.md @@ -1,6 +1,7 @@ # Data Management Migration Resilience Implemented in version: **0.250.071** +Updated in version: **0.250.105** GitHub issue: [#1043](https://github.com/microsoft/simplechat/issues/1043) @@ -82,6 +83,8 @@ Job detail exposes readiness, actual outcomes, preview divergence, keyset cursor Preflight results are saved in the migration state and shown through existing job details and activity history without credentials or source content. +Version 0.250.105 also exposes these checks through the staged browser workflow's **Review** step. The review API accepts the current redacted settings form plus migration plan, resolves stored secret placeholders on the server, and returns only sanitized pass, warning, or blocker results. It combines exhaustive scope/document counts, temporary access probes, partition-key validation, destination coordinator and Search write-gate availability, optional capacity inspection, and the read-only destination inventory preview. Only a ready review receives a short-lived, administrator-bound authorization. Job creation atomically reserves it, releases that exact reservation if durable queueing fails, and the resulting job consumes it before work; blocked, expired, replayed, or changed reviews are rejected. The worker also verifies the reviewed migration-setting fingerprint before execution. The review remains a snapshot, and execution-time worker preflight is authoritative. + ### Temporary Cosmos Capacity The optional **Temporarily increase destination Cosmos capacity during this migration** control raises eligible destination database or dedicated-container throughput to the configured target, capped at **10,000 RU/s**. @@ -94,13 +97,13 @@ The optional **Temporarily increase destination Cosmos capacity during this migr ## Usage -1. Open **Admin Settings > Migration** and configure the destination services. -2. Select the scopes and document surfaces to migrate. -3. Choose **New only**, **Delta / upsert**, or **Mirror with deletions**. Delta and mirror can name a completed baseline job or automatically use the latest compatible baseline. -4. Select **Preview Migration** and review estimated creates, updates, unchanged items, deletes, missing sources, and conflicts. For mirror, type the exact destructive confirmation phrase. -5. Select **Validate Cosmos Migration Access** to confirm the selected source and destination permissions. The worker performs complete Cosmos, Search, and Blob preflight before copying data. -6. Optionally enable the temporary destination Cosmos capacity boost, supply the target subscription and resource group, and leave the target at or below 10,000 RU/s. -7. Execute the migration and open the job log to observe durable inventory, stage state, transfer rates, in-flight Blob bytes, keyset checkpoints, outcome counts, reconciliation readiness, and preview divergence. Download the manifest or failure list when item-level remediation is required. +1. Open **Admin Settings > Data Management > Migration** and complete **Target**. +2. In **Scope**, choose none, selected, or all for each principal type. Search and page selected catalogs without losing choices. +3. In **Content & Options**, choose document surfaces, Search/blobs, synchronization mode, concurrency, retry, and optional capacity behavior. +4. In **Review**, run preflight and resolve blockers for access, partition keys, collisions, locks, compatibility, and capacity. Review estimated creates, updates, unchanged items, deletes, missing sources, and conflicts. +5. In **Confirm**, acknowledge the normalized plan. Mirror mode requires the exact destructive confirmation phrase. +6. Execute the migration and use **Progress** or the full job log to observe durable inventory, stage state, transfer rates, in-flight Blob bytes, keyset checkpoints, outcome counts, reconciliation readiness, and preview divergence. +7. Download the manifest or failure list when item-level remediation is required. 8. Use **Cancel** to request a cooperative stop. Use **Retry** or **Resume** on a failed, canceled, or stale job. The original migration GUID and completed destination markers are retained. ### Recommended Cutover @@ -130,6 +133,7 @@ The optional **Temporarily increase destination Cosmos capacity during this migr - `functional_tests/test_data_management_migration_reconciliation.py` - `functional_tests/test_data_management_migration_manifest.py` - `functional_tests/test_data_management_security_patterns.py` +- `functional_tests/test_data_management_migration_workflow_contract.py` - `ui_tests/test_admin_data_management_settings_ui.py` The focused coverage verifies provenance and source fingerprints, unowned collision rejection, equal-second Cosmos updates, Search transfer and reconciliation beyond 100,000 documents, failed-page cursor retention, Blob mid-transfer heartbeats and pending-to-succeeded verification, mode and baseline validation, two-phase migration-owned mirror deletion, post-cutoff source protection, preview divergence, durable hashed manifests, global coordination, cancellation/retry recovery, route protection, and safe browser rendering. @@ -148,4 +152,5 @@ The focused coverage verifies provenance and source fingerprints, unowned collis ## Version References - Application version updated in `application/single_app/config.py` to `0.250.071`. -- This documentation and the related functional tests use the same implementation version. \ No newline at end of file +- The staged migration workflow and catalog/review contracts were updated in `application/single_app/config.py` version `0.250.105`. +- This documentation and the related functional tests use their corresponding implementation versions. \ No newline at end of file diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index ed1e373fa..b902b880e 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,15 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.105)** + +#### User Interface Enhancements + +* **Reviewed, Scalable Data Migration Workflow** + * Replaced the Admin Data Management migration form with a six-stage Target, Scope, Content & Options, Review, Confirm, and Progress workflow. + * Added server-paginated principal catalogs, exhaustive all-mode counts, persistent cross-page selections, sanitized preflight checks, single-use administrator-bound review authorization, settings-drift protection, separate destructive confirmation, duplicate-submit prevention, and inline durable job recovery controls. + * (Ref: Closes #1097, `functions_data_management.py`, `route_backend_data_management.py`, `admin_settings.html`, `admin_data_management.js`) + ### **(v0.250.103)** #### New Features diff --git a/functional_tests/test_data_management_history_pagination.py b/functional_tests/test_data_management_history_pagination.py index 26aa7c7bd..4a0f220cc 100644 --- a/functional_tests/test_data_management_history_pagination.py +++ b/functional_tests/test_data_management_history_pagination.py @@ -2,9 +2,10 @@ #!/usr/bin/env python3 """ Functional test for Data Management history pagination. -Version: 0.250.104 +Version: 0.250.105 Implemented in: 0.250.103 Updated in: 0.250.104 +Updated in: 0.250.105 This test ensures job history and backup inventory use deterministic, filtered, sanitized Cosmos pages with opaque continuation state and global summaries. @@ -205,6 +206,7 @@ class FakeHistoryPaginationError(ValueError): data_management_module.DATA_MANAGEMENT_OPERATION_RESTORE = "restore" imported_function_names = [ + "create_data_management_migration_review_authorization", "export_data_management_migration_manifest", "generate_data_management_encryption_key", "get_data_management_cosmos_editor_containers", @@ -212,12 +214,16 @@ class FakeHistoryPaginationError(ValueError): "get_data_management_job_detail", "get_data_management_job_progress", "get_data_management_migration_catalog", + "get_data_management_migration_review_fingerprint", "get_data_management_settings", "log_data_management_cosmos_editor_activity", "preview_data_management_migration_plan", "queue_data_management_job", "query_data_management_cosmos_editor_documents", "request_data_management_job_cancellation", + "release_data_management_migration_review_reservation", + "reserve_data_management_migration_review_authorization", + "review_data_management_migration", "retry_data_management_backup_job", "resolve_data_management_migration_manifest_item", "retry_data_management_migration_job", diff --git a/functional_tests/test_data_management_migration_workflow_contract.py b/functional_tests/test_data_management_migration_workflow_contract.py new file mode 100644 index 000000000..98906f56b --- /dev/null +++ b/functional_tests/test_data_management_migration_workflow_contract.py @@ -0,0 +1,566 @@ +# test_data_management_migration_workflow_contract.py +""" +Functional tests for the Admin Data Management migration workflow contract. +Version: 0.250.105 +Implemented in: 0.250.105 + +This test ensures migration catalogs paginate beyond 50 records, bind +continuations to the current search, and report exhaustive all-mode counts. +""" + +import copy +import importlib.util +import json +from pathlib import Path +import sys +import types + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = REPO_ROOT / "application" / "single_app" +MODULE_PATH = APP_ROOT / "functions_data_management.py" +sys.path.insert(0, str(APP_ROOT)) + + +class FakeJobContainer: + """Provide the minimal persistence surface required during import.""" + + def __init__(self): + self.items = {} + + def upsert_item(self, body): + self.items[body["id"]] = copy.deepcopy(body) + return copy.deepcopy(body) + + def create_item(self, body): + created = copy.deepcopy(body) + created["_etag"] = "1" + self.items[created["id"]] = created + return copy.deepcopy(created) + + def read_item(self, item, partition_key): + assert item == partition_key + if item not in self.items: + raise KeyError(item) + return copy.deepcopy(self.items[item]) + + def replace_item( + self, + item, + body, + etag=None, + match_condition=None, + ): + del match_condition + existing = self.items[item] + if etag != existing.get("_etag"): + error = RuntimeError("etag mismatch") + error.status_code = 412 + raise error + replaced = copy.deepcopy(body) + replaced["_etag"] = str(int(existing["_etag"]) + 1) + self.items[item] = replaced + return copy.deepcopy(replaced) + + +class FakeCatalogContainer: + """Evaluate the bounded catalog query contract over in-memory records.""" + + def __init__(self, items): + self.items = [copy.deepcopy(item) for item in items] + + def query_items(self, query, parameters=None, **_kwargs): + parameters = { + parameter["name"]: parameter["value"] + for parameter in (parameters or []) + } + search = str(parameters.get("@catalog_search") or "").lower() + after_id = str(parameters.get("@catalog_after_id") or "") + results = [ + copy.deepcopy(item) + for item in self.items + if ( + not search or + any(search in str(value or "").lower() for value in item.values()) + ) and ( + not after_id or str(item.get("id") or "") > after_id + ) + ] + results.sort(key=lambda item: str(item.get("id") or "")) + if query.startswith("SELECT VALUE COUNT(1)"): + return iter([len(results)]) + return iter(results) + + +class FakeDocumentContainer: + """Return scope and exhaustive document counts.""" + + def __init__(self, counts): + self.counts = dict(counts) + + def query_items(self, query, parameters=None, **_kwargs): + if "@scope_id" not in query: + return iter([sum(self.counts.values())]) + parameter_map = { + parameter["name"]: parameter["value"] + for parameter in (parameters or []) + } + return iter([self.counts.get(parameter_map.get("@scope_id"), 0)]) + + +def load_data_management_module(monkeypatch, users, document_counts): + """Load production code with deterministic migration catalog containers.""" + job_container = FakeJobContainer() + config_module = types.ModuleType("config") + config_module.CLIENTS = {} + config_module.VERSION = "0.250.105" + config_module.cosmos_data_management_jobs_container = job_container + config_module.cosmos_data_management_job_items_container = job_container + config_module.cosmos_settings_container = job_container + config_module.cosmos_user_settings_container = FakeCatalogContainer(users) + config_module.cosmos_user_documents_container = FakeDocumentContainer( + document_counts + ) + config_module.cosmos_groups_container = FakeCatalogContainer([]) + config_module.cosmos_group_documents_container = FakeDocumentContainer({}) + config_module.cosmos_public_workspaces_container = FakeCatalogContainer([]) + config_module.cosmos_public_documents_container = FakeDocumentContainer({}) + monkeypatch.setitem(sys.modules, "config", config_module) + + appinsights_module = types.ModuleType("functions_appinsights") + appinsights_module.log_event = lambda *_args, **_kwargs: None + monkeypatch.setitem(sys.modules, "functions_appinsights", appinsights_module) + + throughput_module = types.ModuleType("functions_cosmos_throughput") + + class FakeCosmosThroughputError(Exception): + pass + + throughput_module.CosmosThroughputError = FakeCosmosThroughputError + throughput_module.get_container_throughput = lambda *_args, **_kwargs: {} + throughput_module.get_database_throughput = lambda *_args, **_kwargs: {} + throughput_module.set_database_throughput = lambda *_args, **_kwargs: {} + monkeypatch.setitem( + sys.modules, + "functions_cosmos_throughput", + throughput_module, + ) + + module_name = "data_management_migration_workflow_contract_test_module" + spec = importlib.util.spec_from_file_location(module_name, MODULE_PATH) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + assert spec and spec.loader + spec.loader.exec_module(module) + monkeypatch.delitem(sys.modules, module_name, raising=False) + return module + + +def build_users(count): + """Create deterministic user catalog records.""" + return [ + { + "id": f"user-{index:03d}", + "display_name": f"Person {index:03d}", + "email": f"person-{index:03d}@example.com", + } + for index in range(count) + ] + + +def test_catalog_paginates_all_records_without_duplicates(monkeypatch): + """Navigate three pages and retain authoritative total metadata.""" + users = build_users(61) + module = load_data_management_module( + monkeypatch, + users, + {user["id"]: 2 for user in users}, + ) + + first_page = module.get_data_management_migration_catalog( + "users", + limit=25, + ) + second_page = module.get_data_management_migration_catalog( + "users", + limit=25, + continuation_token=first_page["continuation_token"], + ) + third_page = module.get_data_management_migration_catalog( + "users", + limit=25, + continuation_token=second_page["continuation_token"], + ) + + combined_ids = [ + item["id"] + for page in (first_page, second_page, third_page) + for item in page["items"] + ] + assert len(combined_ids) == 61 + assert len(set(combined_ids)) == 61 + assert combined_ids == sorted(combined_ids) + assert first_page["total_count"] == 61 + assert first_page["has_more"] is True + assert second_page["has_more"] is True + assert third_page["has_more"] is False + assert third_page["continuation_token"] == "" + + +def test_catalog_search_and_continuation_are_bound_together(monkeypatch): + """Reject a continuation when its target or normalized search changes.""" + users = build_users(61) + module = load_data_management_module(monkeypatch, users, {}) + page = module.get_data_management_migration_catalog( + "users", + search_text="person", + limit=10, + ) + + with pytest.raises( + module.DataManagementSettingsValidationError, + match="current search", + ): + module.get_data_management_migration_catalog( + "users", + search_text="different", + limit=10, + continuation_token=page["continuation_token"], + ) + + with pytest.raises( + module.DataManagementSettingsValidationError, + match="invalid", + ): + module.get_data_management_migration_catalog( + "users", + continuation_token="not-a-valid-token", + ) + + +def test_all_mode_summary_uses_exhaustive_principal_and_document_counts( + monkeypatch, +): + """Ensure all mode does not inherit a catalog page limit.""" + users = build_users(61) + module = load_data_management_module( + monkeypatch, + users, + {user["id"]: 2 for user in users}, + ) + + summary = module.summarize_data_management_migration_plan({ + "migration_plan": { + "users": { + "mode": "all", + "ids": [], + "include_documents": True, + }, + }, + }) + + assert summary["users"]["count"] == 61 + assert summary["users"]["document_count"] == 122 + assert summary["users"]["ids"] == [] + assert summary["users"]["ids_truncated"] is False + + +def test_selected_scope_limit_fails_instead_of_silently_truncating(monkeypatch): + """Reject over-limit selections without changing requested semantics.""" + module = load_data_management_module(monkeypatch, [], {}) + selected_ids = [ + f"user-{index}" + for index in range(module.DATA_MANAGEMENT_MIGRATION_MAX_SELECTED_IDS + 1) + ] + + with pytest.raises( + module.DataManagementSettingsValidationError, + match="cannot exceed", + ): + module.normalize_data_management_migration_plan({ + "migration_plan": { + "users": { + "mode": "selected", + "ids": selected_ids, + }, + }, + }) + + +def test_review_composes_safe_blocking_preflight_results(monkeypatch): + """Expose blockers and counts without returning target credentials.""" + users = build_users(1) + module = load_data_management_module( + monkeypatch, + users, + {"user-000": 3}, + ) + normalized_settings = { + "target_cosmos_authentication_type": "key", + "target_cosmos_endpoint": "https://private.documents.azure.com", + "target_cosmos_key": "never-return-this-key", + "target_cosmos_database_name": "SimpleChat", + "target_ai_search_authentication_type": "key", + "target_ai_search_endpoint": "https://private.search.windows.net", + "target_ai_search_key": "never-return-this-search-key", + "target_enhanced_citations_storage_authentication_type": ( + "managed_identity" + ), + "migration_max_parallel_operations": 8, + "migration_retry_count": 5, + "migration_skip_recent_within_hours": 0, + "migration_temporary_destination_ru_enabled": False, + "migration_temporary_destination_ru": 10000, + } + monkeypatch.setattr( + module, + "_normalize_data_management_settings_from_payload", + lambda _settings: copy.deepcopy(normalized_settings), + ) + monkeypatch.setattr( + module, + "_preflight_target_cosmos_migration_access", + lambda *_args: {"container_count": 1}, + ) + monkeypatch.setattr( + module, + "_preflight_target_ai_search_migration_access", + lambda *_args: {"index_count": 1}, + ) + monkeypatch.setattr( + module, + "_get_target_cosmos_database", + lambda _settings: object(), + ) + monkeypatch.setattr( + module, + "_get_target_data_management_search_write_gate_container", + lambda _database: object(), + ) + monkeypatch.setattr( + module, + "inspect_data_management_target_migration_coordinator", + lambda _container: {"available": True, "active": False}, + ) + monkeypatch.setattr( + module, + "inspect_data_management_search_write_gate", + lambda _container: { + "available": True, + "state": "open", + "active_writer_count": 0, + }, + ) + monkeypatch.setattr( + module, + "preview_data_management_migration_plan", + lambda *_args, **_kwargs: { + "captured_at": "2026-07-30T12:00:00+00:00", + "estimated_outcomes": { + "create_count": 1, + "conflict_count": 2, + "delete_count": 0, + }, + "services": [], + }, + ) + + review = module.review_data_management_migration( + settings=normalized_settings, + migration_plan={ + "users": { + "mode": "selected", + "ids": ["user-000"], + "include_documents": True, + }, + "include_ai_search": True, + "target_ai_search_writes_frozen": True, + }, + ) + + assert review["ready"] is False + assert review["blocker_count"] == 1 + assert review["summary"]["users"]["count"] == 1 + assert review["summary"]["users"]["document_count"] == 3 + collision_check = next( + check + for check in review["checks"] + if check["id"] == "destination_inventory" + ) + assert collision_check["status"] == "block" + serialized_review = json.dumps(review) + assert "never-return-this-key" not in serialized_review + assert "private.documents.azure.com" not in serialized_review + assert "private.search.windows.net" not in serialized_review + + +def test_review_fingerprint_binds_plan_but_not_confirmation_only_state( + monkeypatch, +): + """Allow final destructive confirmation without authorizing plan changes.""" + module = load_data_management_module(monkeypatch, [], {}) + settings = { + "target_cosmos_endpoint": "https://target.documents.azure.com", + "migration_max_parallel_operations": 8, + } + reviewed_plan = { + "users": { + "mode": "selected", + "ids": ["user-001"], + "include_documents": True, + }, + "groups": {"mode": "none", "ids": [], "include_documents": False}, + "public_workspaces": { + "mode": "none", + "ids": [], + "include_documents": False, + }, + "migration_mode": "mirror_with_deletions", + "mirror_deletions_confirmed": False, + } + confirmed_plan = copy.deepcopy(reviewed_plan) + confirmed_plan["mirror_deletions_confirmed"] = True + changed_plan = copy.deepcopy(confirmed_plan) + changed_plan["users"]["ids"] = ["user-002"] + + reviewed_fingerprint = module._migration_review_fingerprint( + settings, + reviewed_plan, + ) + + assert module._migration_review_fingerprint( + settings, + confirmed_plan, + ) == reviewed_fingerprint + assert module._migration_review_fingerprint( + settings, + changed_plan, + ) != reviewed_fingerprint + + +def test_ready_review_authorization_is_admin_bound_and_single_use(monkeypatch): + """Consume a ready review token once and reject replay.""" + module = load_data_management_module(monkeypatch, [], {}) + authorization = ( + module.create_data_management_migration_review_authorization( + "admin-user", + "review-fingerprint", + ) + ) + + reservation = ( + module.reserve_data_management_migration_review_authorization( + authorization["authorization_token"], + "admin-user", + "review-fingerprint", + ) + ) + with pytest.raises( + module.DataManagementSettingsValidationError, + match="reserved or used", + ): + module.reserve_data_management_migration_review_authorization( + authorization["authorization_token"], + "admin-user", + "review-fingerprint", + ) + + assert module.consume_data_management_migration_review_authorization( + authorization["authorization_token"], + "admin-user", + "review-fingerprint", + reservation["reservation_token"], + reservation["job_id"], + ) is True + assert module.consume_data_management_migration_review_authorization( + authorization["authorization_token"], + "admin-user", + "review-fingerprint", + reservation["reservation_token"], + reservation["job_id"], + ) is True + + +def test_review_reservation_can_be_released_after_queue_failure(monkeypatch): + """Restore a ready authorization when durable job creation fails.""" + module = load_data_management_module(monkeypatch, [], {}) + authorization = ( + module.create_data_management_migration_review_authorization( + "admin-user", + "review-fingerprint", + ) + ) + reservation = ( + module.reserve_data_management_migration_review_authorization( + authorization["authorization_token"], + "admin-user", + "review-fingerprint", + ) + ) + + assert module.release_data_management_migration_review_reservation( + authorization["authorization_token"], + reservation["reservation_token"], + ) is True + next_reservation = ( + module.reserve_data_management_migration_review_authorization( + authorization["authorization_token"], + "admin-user", + "review-fingerprint", + ) + ) + assert next_reservation["job_id"] == reservation["job_id"] + + +def test_worker_rejects_settings_changed_after_review(monkeypatch): + """Stop before migration work when queue-time settings no longer match.""" + module = load_data_management_module(monkeypatch, [], {}) + migration_plan = { + "users": { + "mode": "selected", + "ids": ["user-001"], + "include_documents": False, + }, + "groups": {"mode": "none", "ids": [], "include_documents": False}, + "public_workspaces": { + "mode": "none", + "ids": [], + "include_documents": False, + }, + "include_ai_search": False, + "include_source_blobs": False, + "migration_mode": "new_only", + } + reviewed_settings = { + "target_cosmos_endpoint": "https://reviewed.documents.azure.com", + "target_cosmos_authentication_type": "managed_identity", + } + reviewed_fingerprint = module._migration_review_fingerprint( + reviewed_settings, + module.normalize_data_management_migration_plan({ + "migration_plan": migration_plan, + }), + ) + job = { + "id": "11111111-1111-1111-1111-111111111111", + "options": { + "migration_plan": migration_plan, + "review_fingerprint": reviewed_fingerprint, + }, + } + + with pytest.raises( + module.DataManagementSettingsValidationError, + match="settings changed", + ): + module.execute_migration_job( + job, + { + "target_cosmos_endpoint": ( + "https://changed.documents.azure.com" + ), + "target_cosmos_authentication_type": "managed_identity", + }, + ) diff --git a/functional_tests/test_data_management_security_patterns.py b/functional_tests/test_data_management_security_patterns.py index 6ec89dadc..082bcfc15 100644 --- a/functional_tests/test_data_management_security_patterns.py +++ b/functional_tests/test_data_management_security_patterns.py @@ -2,11 +2,12 @@ # test_data_management_security_patterns.py """ Functional test for Data Management security patterns. -Version: 0.250.104 +Version: 0.250.105 Implemented in: 0.241.211 Updated in: 0.250.102 Updated in: 0.250.103 Updated in: 0.250.104 +Updated in: 0.250.105 This test ensures Data Management admin routes require authenticated admin access, secrets stay redacted in frontend responses, and the admin browser @@ -21,6 +22,8 @@ results-pane scroll refinement. Version 0.250.076 adds bounded parallel backup checkpoints, source capacity recovery, generic admin cancellation/retry controls, and latest-only sidecar state sanitization. +Version 0.250.103 verifies paginated migration catalogs and sanitized +server-owned review results. """ import ast @@ -71,7 +74,7 @@ def test_version_and_container_registration(): """Validate the Data Management version and Cosmos job container registrations.""" config_source = read_text(CONFIG_FILE) - assert 'VERSION = "0.250.104"' in config_source + assert 'VERSION = "0.250.105"' in config_source assert 'cosmos_data_management_jobs_container_name = "data_management_jobs"' in config_source assert 'partition_key=PartitionKey(path="/id")' in config_source assert 'cosmos_data_management_job_items_container_name = "data_management_job_items"' in config_source @@ -91,7 +94,7 @@ def test_version_and_container_registration(): def test_admin_routes_require_login_admin_and_swagger_security(): """Validate every Data Management route has the required admin security stack.""" routes = route_functions_with_decorators() - assert len(routes) == 23 + assert len(routes) == 24 for function_name, decorators in routes: assert "swagger_route" in decorators, f"{function_name} missing swagger_route" @@ -99,6 +102,22 @@ def test_admin_routes_require_login_admin_and_swagger_security(): assert "admin_required" in decorators, f"{function_name} missing admin_required" source = read_text(ROUTE_FILE) + route_tree = ast.parse(source) + create_job_function = next( + node + for node in ast.walk(route_tree) + if isinstance(node, ast.FunctionDef) and + node.name == "create_admin_data_management_job" + ) + assert any( + isinstance(node, ast.Assign) and + any( + isinstance(target, ast.Name) and + target.id == "review_reservation" + for target in node.targets + ) + for node in create_job_function.body + ) assert 'from swagger_wrapper import get_auth_security, swagger_route' in source assert '/api/admin/data-management/settings' in source assert '/api/admin/data-management/jobs' in source @@ -106,6 +125,7 @@ def test_admin_routes_require_login_admin_and_swagger_security(): assert '/api/admin/data-management/backups' in source assert '/api/admin/data-management/migration/catalog/' in source assert '/api/admin/data-management/migration/summary' in source + assert '/api/admin/data-management/migration/review' in source assert '/api/admin/data-management/target/cosmos/test' in source assert '/api/admin/data-management/target/search/test' in source assert '/api/admin/data-management/target/enhanced-citation-storage/test' in source @@ -313,13 +333,13 @@ def test_admin_javascript_uses_safe_dom_patterns(): 'openKeyVaultSettings', 'buildMigrationPlan()', 'queueMigration(false)', - 'loadMigrationCatalog(targetType)', - 'renderMigrationSummary(data.summary || {}, data.preview || null)', + 'loadMigrationCatalog(targetType, "reset")', + 'renderMigrationSummary(review.summary || {}, review.preview || null)', 'testTargetCosmos', 'testMigrationAccess', 'testTargetSearch', 'testTargetEnhancedCitationStorage', - 'Migration preview refreshed.', + 'Migration preflight review is ready for confirmation.', 'setMigrationTargetVisibility()', 'updateSourceBlobBackupAvailability(settings)', 'updateKeyStorageExperience(settings)', @@ -391,8 +411,8 @@ def test_admin_ui_exposes_data_management_without_external_assets(): 'id="data-management-backup-inventory-section"', 'We suggest not running backups, restores, or migrations during your operational business hours.', '

Backup

', - '

Migration

', - '
Target Cosmos Database
', + 'id="data-management-migration-title"', + 'id="data-management-target-cosmos-heading"', 'id="data_management_full_frequency"', 'id="data_management_scheduled_time_utc" value="03:00"', 'id="data_management_partial_enabled"', @@ -412,7 +432,7 @@ def test_admin_ui_exposes_data_management_without_external_assets(): 'id="data_management_migration_mirror_confirmation_phrase"', 'id="data-management-migration-search-write-freeze"', 'id="data_management_migration_target_search_writes_frozen"', - 'I confirm external destination AI Search writers are frozen for this migration', + 'I confirm external destination AI Search writers are frozen', 'id="data-management-test-target-cosmos-btn"', 'id="data-management-target-ai-search-section"', 'id="data-management-test-target-search-btn"', @@ -420,7 +440,7 @@ def test_admin_ui_exposes_data_management_without_external_assets(): 'id="data-management-test-target-ec-storage-btn"', 'id="data-management-migration-workflow-section"', 'id="data-management-test-migration-access-btn"', - 'Validate Cosmos Migration Access', + 'Validate Cosmos Access', 'id="data_management_migration_max_parallel_operations"', 'id="data_management_backup_max_parallel_operations"', 'id="data_management_backup_retry_count"', @@ -504,10 +524,19 @@ def test_admin_ui_exposes_data_management_without_external_assets(): assert 'requestGeneration !== state.requestGeneration' in read_text(ADMIN_JS) assert 'params.set("continuation_token", state.currentToken);' in read_text(ADMIN_JS) assert 'DataManagementSettingsValidationError as exc' in read_text(ROUTE_FILE) - assert 'except DataManagementHistoryPaginationError:' in read_text(ROUTE_FILE) - assert 'DATA_MANAGEMENT_HISTORY_VALIDATION_ERROR' in read_text(ROUTE_FILE) - assert 'except DataManagementHistoryPaginationError as exc:' not in read_text(ROUTE_FILE) - assert 'get_data_management_migration_catalog(target_type, search_text=search, limit=limit)' in read_text(ROUTE_FILE) + route_source = read_text(ROUTE_FILE) + assert 'continuation_token=continuation_token' in route_source + assert '@bp.route("/api/admin/data-management/migration/review", methods=["POST"])' in route_source + assert 'review_data_management_migration(' in route_source + assert '"workflow_step": "review"' in route_source + assert 'hmac.compare_digest(' in route_source + assert 'get_data_management_migration_review_fingerprint(' in route_source + assert 'create_data_management_migration_review_authorization(' in route_source + assert 'reserve_data_management_migration_review_authorization(' in route_source + assert 'release_data_management_migration_review_reservation(' in route_source + assert 'except DataManagementHistoryPaginationError:' in route_source + assert 'DATA_MANAGEMENT_HISTORY_VALIDATION_ERROR' in route_source + assert 'except DataManagementHistoryPaginationError as exc:' not in route_source assert 'data-management-restore-dry-run-btn' not in template assert 'data-management-migration-dry-run-btn' not in template admin_settings_js = read_text(APP_ROOT / "static" / "js" / "admin" / "admin_settings.js") diff --git a/ui_tests/test_admin_data_management_settings_ui.py b/ui_tests/test_admin_data_management_settings_ui.py index b3656f47b..a6fa14313 100644 --- a/ui_tests/test_admin_data_management_settings_ui.py +++ b/ui_tests/test_admin_data_management_settings_ui.py @@ -1,12 +1,13 @@ # test_admin_data_management_settings_ui.py """ UI test for Admin Settings Data Management controls. -Version: 0.250.104 +Version: 0.250.105 Implemented in: 0.241.211 Updated in: 0.241.221 Updated in: 0.250.102 Updated in: 0.250.103 Updated in: 0.250.104 +Updated in: 0.250.105 This test ensures admins can discover the Data Management tab, see the operational-business-hours warning, and access the backup, encryption, @@ -18,6 +19,8 @@ reconciliation, and the external target Search writer freeze acknowledgement. Version 0.250.076 adds bounded parallel backup and source capacity controls. Version 0.250.102 adds independently bounded source-blob transfer controls. +Version 0.250.103 adds the staged migration workflow, scalable catalogs, +server-owned review, confirmation gating, and inline durable job progress. """ import json @@ -110,6 +113,22 @@ def test_admin_data_management_controls_render_from_template(): "data_management_target_ec_connection_string", "data-management-test-target-ec-storage-btn", "data-management-migration-workflow-section", + "data-management-migration-readiness", + "data-management-migration-step-error", + "data-management-migration-scope-total", + "data-management-migration-review-empty", + "data-management-migration-review-checks", + "data-management-migration-confirm-summary", + "data_management_migration_final_confirmation", + "data-management-migration-start-new-btn", + "data-management-migration-view-job-btn", + "data-management-migration-workflow-progress", + "data-management-migration-workflow-progress-summary", + "data-management-migration-workflow-progress-state", + "data-management-migration-workflow-progress-actions", + "data-management-migration-back-btn", + "data-management-migration-next-btn", + "data-management-migration-step-position", "data-management-test-migration-access-btn", "data-management-migration-mode-section", "data_management_migration_mode_new_only", @@ -212,7 +231,7 @@ def test_admin_data_management_controls_render_from_template(): assert 'id="data-management" role="tabpanel" aria-labelledby="data-management-tab" data-testid="data-management-tab-pane" data-ignore-settings-change="true"' in template assert 'id="data-management-save-settings-btn" disabled aria-disabled="true"' in template assert '

Backup

' in template - assert '

Migration

' in template + assert 'id="data-management-migration-title"' in template assert "Cosmos DB JSON Editor" in template assert "Query results and the JSON editor open in a modal" in template assert "modal-xl modal-dialog-scrollable" in template @@ -233,11 +252,11 @@ def test_admin_data_management_controls_render_from_template(): assert 'Protection' in template assert 'Backup Contents' in template assert 'Storage and Manifest' in template - assert '
Target Cosmos Database
' in template - assert '
Target Search
' in template - assert '
Target Enhanced Citation Storage
' in template - assert '
Migration Workflow
' in template - assert "Use migration when moving SimpleChat data into another SimpleChat environment" in template + assert 'id="data-management-target-cosmos-heading"' in template + assert 'id="data-management-target-search-heading"' in template + assert 'id="data-management-target-storage-heading"' in template + assert 'aria-label="Migration workflow"' in template + assert "Move SimpleChat data through a reviewed, recoverable environment transfer." in template assert "Full backups run on the selected cadence; partial backups run daily only." in template assert "Advanced backup scope" in template assert "Modify them at your own risk" in template @@ -246,15 +265,29 @@ def test_admin_data_management_controls_render_from_template(): assert "Cosmos Backup Performance" in template assert "Temporary source max RU/s" in template assert "continue_without_boost" in template - assert "For managed identity, assign this App Service identity Cosmos DB Data Contributor" in template + assert "Managed identity requires Cosmos DB Data Contributor" in template assert "Paste a connection string to save or replace it" in template assert 'id="data-management-connection-string-status"' in template assert 'id="data_management_target_cosmos_database" value="SimpleChat" readonly aria-readonly="true"' in template assert 'max="10000"' in template - assert 'Validate Cosmos Migration Access' in template + assert 'Validate Cosmos Access' in template assert 'role="radiogroup" aria-label="Migration synchronization mode"' in template assert "MIRROR WITH DELETIONS" in template - assert "I confirm external destination AI Search writers are frozen for this migration" in template + assert "I confirm external destination AI Search writers are frozen" in template + assert 'data-migration-step-button="target"' in template + assert 'data-migration-step-button="scope"' in template + assert 'data-migration-step-button="options"' in template + assert 'data-migration-step-button="review"' in template + assert 'data-migration-step-button="confirm"' in template + assert 'data-migration-step-button="progress"' in template + assert 'data-migration-mode-option="users"' in template + assert 'data-migration-page-next="users"' in template + assert 'data-migration-page-previous="users"' in template + assert 'data-migration-all-count="users"' in template + assert ".migration-stepper" in css_source + assert ".migration-catalog-workspace" in css_source + assert ".migration-review-check" in css_source + assert "@media (max-width: 767.98px)" in css_source assert 'setStorageAuthVisibility' in js_source assert 'updateConnectionStringStatus' in js_source assert 'updateSourceBlobBackupAvailability' in js_source @@ -272,7 +305,19 @@ def test_admin_data_management_controls_render_from_template(): assert 'backup_temporary_source_ru_enabled' in js_source assert 'buildMigrationPlan' in js_source assert 'queueMigration(false)' in js_source - assert 'loadMigrationCatalog(targetType)' in js_source + assert 'loadMigrationCatalog(targetType, "reset")' in js_source + assert 'migrationCatalogStates' in js_source + assert 'requestGeneration' in js_source + assert 'continuation_token=' in js_source + assert 'runMigrationReview' in js_source + assert '/api/admin/data-management/migration/review' in js_source + assert 'reviewStale' in js_source + assert 'reviewRequestGeneration' in js_source + assert 'submissionInFlight' in js_source + assert 'submissionAccepted' in js_source + assert 'resetMigrationWorkflowForNewRun' in js_source + assert 'attachMigrationWorkflowJob' in js_source + assert 'refreshMigrationWorkflowProgress' in js_source assert 'loadCosmosEditorContainers' in js_source assert 'queryCosmosEditorDocuments(false)' in js_source assert 'cosmosEditorContinuationToken' in js_source @@ -297,7 +342,9 @@ def test_admin_data_management_controls_render_from_template(): assert 'updateMigrationSearchWriteFreezeVisibility' in js_source assert 'target_ai_search_writes_frozen' in js_source assert 'createMigrationPreviewOutcomes' in js_source - assert 'include_inventory: Boolean(showSuccess)' in js_source + assert 'settings: collectSettings()' in js_source + assert 'review_fingerprint: migrationWorkflowState.review?.review_fingerprint' in js_source + assert 'review_authorization_token: migrationWorkflowState.review?.authorization_token' in js_source assert 'Stage ${formatNumber(displayedStage)} of ${formatNumber(totalSteps)}' in js_source assert 'Active stage' in js_source assert 'Migration stage is active; measured throughput is shown below.' in js_source @@ -305,7 +352,7 @@ def test_admin_data_management_controls_render_from_template(): assert 'Retry failures' in js_source assert 'testTargetSearch' in js_source assert 'testTargetEnhancedCitationStorage' in js_source - assert 'Migration preview refreshed.' in js_source + assert 'Migration preflight review is ready for confirmation.' in js_source assert 'Enhanced Citations is off, so source document blob backups are unavailable.' in js_source assert 'Stored connection string saved. You can test storage without re-entering it.' in js_source assert 'target_cosmos_database_name: targetCosmosDatabaseName' in js_source @@ -325,9 +372,8 @@ def test_admin_data_management_controls_render_from_template(): assert "data-management-migration-dry-run-btn" not in template -@pytest.mark.ui -def test_admin_data_management_tab_browser_workflow(): - """Validate the rendered admin tab in an authenticated browser session when configured.""" +def _run_admin_migration_browser_workflow(viewport): + """Exercise the staged workflow with deterministic API responses.""" if not BASE_URL: pytest.skip("Set SIMPLECHAT_UI_BASE_URL to run this UI test.") if not STORAGE_STATE or not Path(STORAGE_STATE).exists(): @@ -337,8 +383,172 @@ def test_admin_data_management_tab_browser_workflow(): playwright_context = sync_playwright().start() browser = playwright_context.chromium.launch() - context = browser.new_context(storage_state=STORAGE_STATE, viewport={"width": 1440, "height": 900}) + context = browser.new_context(storage_state=STORAGE_STATE, viewport=viewport) page = context.new_page() + job_submissions = [] + + def fulfill_json(route, payload, status=200): + route.fulfill( + status=status, + content_type="application/json", + body=json.dumps(payload), + ) + + def handle_data_management_api(route): + request = route.request + url = request.url + if "/migration/catalog/users" in url: + is_second_page = "continuation_token=next-page" in url + fulfill_json(route, { + "success": True, + "type": "users", + "items": [{ + "id": "user-002" if is_second_page else "user-001", + "label": "Second User" if is_second_page else "Admin User", + "description": ( + "second@example.com" + if is_second_page + else "admin@example.com" + ), + "document_count": 4 if is_second_page else 7, + }], + "total_count": 51, + "page_size": 25, + "has_more": not is_second_page, + "continuation_token": "" if is_second_page else "next-page", + }) + return + if "/migration/catalog/" in url: + fulfill_json(route, { + "success": True, + "items": [], + "total_count": 0, + "page_size": 1, + "has_more": False, + "continuation_token": "", + }) + return + if url.endswith("/api/admin/data-management/migration/review"): + fulfill_json(route, { + "success": True, + "review": { + "ready": True, + "blocker_count": 0, + "warning_count": 1, + "review_fingerprint": "review-fingerprint", + "authorization_token": "review-authorization-token", + "authorization_expires_at": "2026-07-30T12:15:00+00:00", + "summary": { + "users": { + "mode": "all", + "count": 51, + "document_count": 357, + "include_documents": True, + }, + "groups": { + "mode": "none", + "count": 0, + "document_count": 0, + "include_documents": False, + }, + "public_workspaces": { + "mode": "none", + "count": 0, + "document_count": 0, + "include_documents": False, + }, + "include_ai_search": False, + "include_source_blobs": False, + "migration_mode": "new_only", + }, + "preview": { + "captured_at": "2026-07-30T12:00:00+00:00", + "estimated_outcomes": { + "create_count": 408, + "update_count": 0, + "unchanged_count": 0, + "delete_count": 0, + "not_applicable_count": 0, + "missing_count": 0, + "conflict_count": 0, + }, + }, + "checks": [ + { + "id": "scope", + "label": "Migration scope", + "workflow_step": "scope", + "status": "pass", + "summary": "51 principal scopes are included.", + }, + { + "id": "source_blob_access", + "label": "Enhanced Citation source blob readiness", + "workflow_step": "options", + "status": "warning", + "summary": "Enhanced Citation source blobs are excluded.", + }, + ], + }, + }) + return + if ( + url.endswith("/api/admin/data-management/settings") and + request.method == "PUT" + ): + settings = request.post_data_json or {} + settings.update({ + "enhanced_citations_enabled": True, + "key_vault_secret_storage_enabled": False, + "key_vault_name_configured": False, + "encryption_key_storage": "not_configured", + }) + fulfill_json(route, {"success": True, "settings": settings}) + return + if ( + url.endswith("/api/admin/data-management/jobs") and + request.method == "POST" + ): + job_submissions.append(request.post_data_json) + fulfill_json(route, { + "success": True, + "job": { + "id": "11111111-1111-1111-1111-111111111111", + "operation": "migration", + "status": "queued", + "created_at": "2026-07-30T12:00:00+00:00", + "progress": { + "completed_steps": 0, + "total_steps": 10, + "percent_complete": 0, + "current_step": "plan", + }, + "can_cancel": True, + "can_retry": False, + }, + }, status=202) + return + if url.endswith("/progress"): + fulfill_json(route, { + "success": True, + "job": { + "id": "11111111-1111-1111-1111-111111111111", + "operation": "migration", + "status": "running", + "progress": { + "completed_steps": 2, + "total_steps": 10, + "percent_complete": 20, + "current_step": "preflight", + }, + "can_cancel": True, + "can_retry": False, + }, + }) + return + route.fallback() + + page.route("**/api/admin/data-management/**", handle_data_management_api) try: response = page.goto(f"{BASE_URL}/admin/settings#data-management", wait_until="networkidle") @@ -362,27 +572,74 @@ def test_admin_data_management_tab_browser_workflow(): expect(page.get_by_role("button", name="Advanced backup scope")).to_be_visible() expect(page.locator("#data_management_target_cosmos_database")).to_have_value("SimpleChat") expect(page.locator("#data_management_target_cosmos_database")).to_have_attribute("readonly", "") - expect(page.locator("#data_management_migration_max_parallel_operations")).to_be_visible() expect(page.locator("#data_management_backup_max_parallel_operations")).to_be_visible() expect(page.locator("#data_management_backup_retry_count")).to_be_visible() expect(page.locator("#data_management_backup_blob_max_parallel_operations")).to_be_visible() expect(page.locator("#data_management_backup_blob_chunk_size_mib")).to_be_visible() expect(page.locator("#data_management_backup_blob_retry_count")).to_be_visible() expect(page.get_by_label("Temporarily increase local source Cosmos capacity for this backup")).to_be_visible() - expect(page.locator("#data_management_migration_retry_count")).to_be_visible() - expect(page.locator("#data_management_migration_skip_recent_within_hours")).to_be_visible() - expect(page.get_by_label("New only")).to_be_checked() - expect(page.locator("#data-management-migration-baseline-field")).to_have_class(re.compile(r"\bd-none\b")) - page.get_by_label("Delta / upsert").check() - expect(page.locator("#data-management-migration-baseline-field")).not_to_have_class(re.compile(r"\bd-none\b")) - page.get_by_label("Mirror with deletions").check() - expect(page.locator("#data-management-migration-mirror-confirmation")).to_be_visible() - expect(page.get_by_label("Temporarily increase destination Cosmos capacity during this migration")).to_be_visible() - expect(page.get_by_role("button", name="Validate Cosmos Migration Access")).to_be_visible() expect(page.locator("#data-management-target-ai-search-section")).to_be_visible() expect(page.locator("#data-management-test-target-search-btn")).to_be_visible() expect(page.locator("#data-management-migration-workflow-section")).to_be_visible() - expect(page.locator("#data-management-execute-migration-btn")).to_be_visible() + expect(page.locator('[data-migration-step-panel="target"]')).to_be_visible() + expect(page.locator('[data-migration-step-panel="scope"]')).to_have_class(re.compile(r"\bd-none\b")) + if not page.locator("#data_management_target_cosmos_endpoint").input_value(): + page.locator("#data_management_target_cosmos_endpoint").fill( + "https://target.documents.azure.com:443/" + ) + page.locator("#data-management-migration-next-btn").click() + expect(page.locator('[data-migration-step-panel="scope"]')).to_be_visible() + page.locator('[data-migration-mode-option="users"][value="selected"]').check() + expect(page.locator("#data-management-migration-users-available")).to_contain_text( + "Admin User" + ) + page.locator("#data-management-migration-users-available").get_by_role( + "button" + ).click() + expect(page.locator("#data-management-migration-users-selected")).to_contain_text( + "Admin User" + ) + page.locator('[data-migration-page-next="users"]').click() + expect(page.locator("#data-management-migration-users-available")).to_contain_text( + "Second User" + ) + expect(page.locator("#data-management-migration-users-selected")).to_contain_text( + "Admin User" + ) + page.locator('[data-migration-mode-option="users"][value="all"]').check() + expect(page.locator('[data-migration-all-count="users"]')).to_contain_text( + "51 Users included" + ) + page.locator("#data_management_migration_users_documents").check() + page.locator("#data-management-migration-next-btn").click() + expect(page.locator('[data-migration-step-panel="options"]')).to_be_visible() + expect(page.get_by_label("New only")).to_be_checked() + page.locator("#data_management_migration_include_ai_search").uncheck() + page.locator("#data-management-migration-next-btn").click() + expect(page.locator('[data-migration-step-panel="review"]')).to_be_visible() + page.locator("#data-management-migration-preview-btn").click() + expect(page.locator("#data-management-migration-readiness")).to_contain_text( + "Ready with 1 warning" + ) + expect(page.locator("#data-management-migration-review-checks")).to_contain_text( + "Migration scope" + ) + page.locator("#data-management-migration-next-btn").click() + expect(page.locator('[data-migration-step-panel="confirm"]')).to_be_visible() + page.locator("#data_management_migration_final_confirmation").check() + expect(page.locator("#data-management-execute-migration-btn")).to_be_enabled() + page.locator("#data-management-execute-migration-btn").evaluate( + "(button) => { button.click(); button.click(); }" + ) + expect(page.locator('[data-migration-step-panel="progress"]')).to_be_visible() + expect(page.locator("#data-management-migration-workflow-progress")).to_be_visible() + expect(page.locator("#data-management-migration-workflow-progress-summary")).to_contain_text( + "11111111-1111-1111-1111-111111111111" + ) + assert len(job_submissions) == 1 + workflow_box = page.locator("#data-management-migration-section").bounding_box() + assert workflow_box + assert workflow_box["width"] <= viewport["width"] expect(page.locator("#data-management-cosmos-editor-section")).to_be_visible() expect(page.locator("#data-management-cosmos-editor-locked-message")).to_be_visible() expect(page.locator("#data-management-cosmos-editor-workspace")).to_have_class(re.compile(r"\bd-none\b")) @@ -409,6 +666,18 @@ def test_admin_data_management_tab_browser_workflow(): playwright_context.stop() +@pytest.mark.ui +def test_admin_data_management_tab_browser_workflow(): + """Validate the full migration workflow at a desktop viewport.""" + _run_admin_migration_browser_workflow({"width": 1440, "height": 900}) + + +@pytest.mark.ui +def test_admin_data_management_tab_mobile_workflow(): + """Validate the full migration workflow at a mobile viewport.""" + _run_admin_migration_browser_workflow({"width": 390, "height": 844}) + + @pytest.mark.ui def test_admin_data_management_history_pagination_browser_workflow(): """Validate authenticated filters and opaque previous/next navigation.""" @@ -561,4 +830,4 @@ def handle_backups(route): finally: context.close() browser.close() - playwright_context.stop() \ No newline at end of file + playwright_context.stop()