diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index a20dee787..c478a0793 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -82,6 +82,11 @@ input would otherwise save to a top-level key nothing reads. See ``_apply_nested_paths``. +``keywords`` + Extra words the V2 page search matches on. A bespoke component has one label and + one help text but can hold a whole workbench, so without these a search for + "retention" would not find the backup schedule that contains it. + ``on_enable`` Companion values a switch sets when it is turned on, as ``{"set": {key: value}, "when": }``. Enhanced extraction is the @@ -94,7 +99,10 @@ result. A value named in the same save always wins. The Appearance, Agents & Actions, Chat, Data Lifecycle, Knowledge, Operations, Scale, -Workflow, Workspaces and Security groups are described in full. Sections with no entry here fall back to the V2 surface's +Workflow, Workspaces and Security groups are described in full. Backup & Recovery is +described by one component per section, because its settings are a separate document +saved through the data-management API and the rest of it is workbenches over that same +API. Sections with no entry here fall back to the V2 surface's ``enable_*`` scan, so undescribed groups keep working exactly as they did. A handful of individual fields outside those groups are also declared: that scan places a key by guessing from shared word stems, and declaring a field is the @@ -7741,6 +7749,138 @@ ], }, ], + # Backup & Recovery. Every section is a bespoke component: the data-management + # settings live in their own Cosmos document (`backup_settings`) and are saved through + # /api/admin/data-management/settings, and the inventory, jobs, migration and editor + # are workbenches over that same admin API rather than settings at all. The V2 page + # counts the components' unsaved settings in its Save bar and saves them after the + # main settings. `keywords` lets the page search find a section by what it contains. + "data-management-readiness-section": [ + { + "type": "component", + "component": "data-management-readiness", + "label": "Backup readiness", + "help": ( + "What is in place for backups, restore and migration, with a guide for " + "each." + ), + "keywords": [ + "start here", "readiness", "checklist", "guide", "setup", "ru boost", + "permissions", "business hours", + ], + }, + ], + "data-management-backup-section": [ + { + "type": "component", + "component": "data-management-backup-runs", + "label": "Run a backup", + "help": ( + "Queue a full or partial backup now, and tune how backups load Cosmos DB " + "and copy source files." + ), + "keywords": [ + "backup now", "full backup", "partial backup", "queue", "performance", + "concurrency", "retry", "throughput", "ru boost", "chunk size", + "file transfers", + ], + }, + ], + "data-management-schedule-section": [ + { + "type": "component", + "component": "data-management-schedule", + "label": "Backup schedule", + "help": "When backups run, how long they are kept, and what they include.", + "keywords": [ + "schedule", "frequency", "daily", "weekly", "retention", + "delete backups after", "partial backups", "low impact", "scope", + "cosmos", "ai search", "source files", "blobs", + ], + }, + ], + "data-management-storage-section": [ + { + "type": "component", + "component": "data-management-storage", + "label": "Backup storage", + "help": "The dedicated Azure Storage account and container backups are written to.", + "keywords": [ + "storage", "blob", "container", "connection string", "managed identity", + "prefix", "test storage", + ], + }, + ], + "data-management-encryption-section": [ + { + "type": "component", + "component": "data-management-encryption", + "label": "Backup encryption", + "help": "Whether backup files are encrypted, and where the encryption key is kept.", + "keywords": ["encryption", "encrypt", "key", "generate key", "key vault"], + }, + ], + "data-management-migration-section": [ + { + "type": "component", + "component": "data-management-migration", + "label": "Migration", + "help": ( + "Move selected users, groups and public workspaces, with their documents, " + "into another SimpleChat environment." + ), + "keywords": [ + "migrate", "migration", "destination", "target", "users", "groups", + "public workspaces", "preflight", "review", "mirror", "catch up", + "baseline", "ru boost", "writer freeze", + ], + }, + ], + "data-management-backup-inventory-section": [ + { + "type": "component", + "component": "data-management-backup-inventory", + "label": "Backup inventory and restore", + "help": ( + "Completed backups, retention cleanup, and restoring a backup into the " + "destination environment." + ), + "keywords": [ + "restore", "inventory", "backups", "retention cleanup", "delete backup", + "overwrite", "create only", + ], + }, + ], + "data-management-cosmos-editor-section": [ + { + "type": "component", + "component": "data-management-cosmos-editor", + "label": "Cosmos DB editor", + "help": ( + "Query SimpleChat's Cosmos DB containers and repair one document at a " + "time, with ETag protection." + ), + "keywords": [ + "cosmos", "json", "editor", "query", "document", "repair", "etag", + "partition key", + ], + }, + ], + "data-management-jobs-section": [ + { + "type": "component", + "component": "data-management-jobs", + "label": "Job history", + "help": ( + "Backup, restore and migration jobs, with live progress, retry, resume, " + "cancel and migration manifests." + ), + "keywords": [ + "jobs", "history", "progress", "cancel", "retry", "resume", "manifest", + "dry run", + ], + }, + ], # ------------------------------------------------------------------ # Operations group: Control Center and Logging & Health. Sections and order # follow admin_settings_nav.py, and every control follows the V1 diff --git a/application/single_app/admin_settings_nav.py b/application/single_app/admin_settings_nav.py index 0b8747e42..2a87a2ba7 100644 --- a/application/single_app/admin_settings_nav.py +++ b/application/single_app/admin_settings_nav.py @@ -504,7 +504,7 @@ "label": "Restore", "icon": "bi-box-seam", "sections": [ - {"id": "data-management-backup-inventory-section", "label": "Backup Inventory & Restore", "icon": "bi-box-seam"}, + {"id": "data-management-backup-inventory-section", "label": "Backup Inventory & Restore", "icon": "bi-box-seam"}, ], }, { diff --git a/application/single_app/config.py b/application/single_app/config.py index 3c12f2205..877179baf 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.274" +VERSION = "0.261.275" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/v2_ui/src/components/admin/dataManagement/DataManagementBackupInventory.tsx b/application/v2_ui/src/components/admin/dataManagement/DataManagementBackupInventory.tsx new file mode 100644 index 000000000..ab7f74ad1 --- /dev/null +++ b/application/v2_ui/src/components/admin/dataManagement/DataManagementBackupInventory.tsx @@ -0,0 +1,793 @@ +// DataManagementBackupInventory.tsx +// Backup Inventory & Restore: the workbench keeps the risky restore and cleanup actions beside +// the backup evidence that makes them safe to choose. + +import { useEffect, useRef, useState, type KeyboardEvent } from 'react'; +import { clsx } from 'clsx'; +import { + BriefcaseBusiness, + CalendarClock, + DatabaseBackup, + FileArchive, + Lock, + RefreshCw, + RotateCcw, + Trash2, +} from 'lucide-react'; +import { + deleteBackup, + errorMessage, + HISTORY_PAGE_SIZES, + listBackups, + readHistoryFailure, + runRetentionCleanup, + type BackupGlobalSummary, + type BackupPage, + type BackupRow, + type HistoryFailure, + type RetentionCleanupResult, +} from '../../../lib/dataManagement'; +import { + computeRetentionDays, + DM_SECTION_IDS, + FIRST_PAGE, + formatBackupType, + formatBytes, + formatDateTime, + formatNumber, + pagerAfterLoad, + pagerBack, + pagerForward, + pagerPageNumber, + readDmValues, + validateHistoryDateRange, +} from '../../../lib/dataManagementLogic'; +import { currentEpoch, isCurrentEpoch, useDataManagementStore } from '../../../stores/dataManagementStore'; +import { toast } from '../../../stores/toastStore'; +import { ConfirmDialog } from '../../ui/ConfirmDialog'; +import { GlassButton, Skeleton } from '../../ui/primitives'; +import { + DmChip, + DmDateFilter, + DmEmpty, + DmFilterSelect, + DmIntro, + DmMetricGrid, + DmNotice, + DmPager, + DmStatusPill, + DmWorkbench, + isStacked, + useVisibleOnce, + type DmCardProps, +} from './DmShared'; +import { useSaveFirst } from './useSaveFirst'; +import { DataManagementRestoreDialog } from './DataManagementRestoreDialog'; + +const STATUS_OPTIONS = [ + ['available', 'Available'], + ['', 'All statuses'], + ['completed', 'Completed'], + ['completed_with_warnings', 'Completed with warnings'], + ['queued', 'Queued'], + ['running', 'Running'], + ['failed', 'Failed'], + ['canceled', 'Canceled'], +] as const; + +const RUN_TYPE_OPTIONS = [ + ['all', 'Scheduled and manual'], + ['scheduled', 'Scheduled'], + ['manual', 'Manual'], +] as const; + +const PAGE_SIZE_OPTIONS = HISTORY_PAGE_SIZES.map((size) => [String(size), String(size)] as const); + +function count(value: unknown): number { + const number = Number(value ?? 0); + return Number.isFinite(number) ? number : 0; +} + +function text(value: unknown, fallback = ''): string { + return typeof value === 'string' && value.trim() ? value.trim() : fallback; +} + +function runType(backup: BackupRow): string { + return backup.scheduled ? 'Scheduled' : 'Manual'; +} + +function contentsLabel(backup: BackupRow): string { + return `${formatNumber(backup.record_count)} records / ${formatNumber(backup.blob_count)} blobs`; +} + +function warningLabel(backup: BackupRow): string { + const warnings = count(backup.warning_count); + return warnings ? `${formatNumber(warnings)} warning${warnings === 1 ? '' : 's'}` : 'No warnings'; +} + +function canRestore(backup: BackupRow | null): boolean { + return Boolean( + backup?.id && + backup.manifest_path && + (backup.status === 'completed' || backup.status === 'completed_with_warnings'), + ); +} + +function cleanupResultMessage(result: RetentionCleanupResult): string { + const deleted = count(result.deleted_count); + return deleted + ? `Deleted ${formatNumber(deleted)} expired backup${deleted === 1 ? '' : 's'}.` + : 'No expired backups were deleted.'; +} + +function SummaryTile({ + label, + description, + value, + active, + icon, + onClick, +}: { + label: string; + description: string; + value: unknown; + active: boolean; + icon: JSX.Element; + onClick: () => void; +}) { + return ( + + ); +} + +export function DataManagementBackupInventory({ help, onNavigate, disabled }: DmCardProps) { + const rootRef = useRef(null); + const detailRef = useRef(null); + const listRef = useRef(null); + const visible = useVisibleOnce(rootRef); + const { ensure, dialog } = useSaveFirst(); + + const inventory = useDataManagementStore((state) => state.inventory); + const backupsRevision = useDataManagementStore((state) => state.backupsRevision); + const updateInventory = useDataManagementStore((state) => state.updateInventory); + + const [page, setPage] = useState({ summary: {}, backups: [], pagination: {} }); + const [loading, setLoading] = useState(false); + const [failure, setFailure] = useState(null); + const [refreshNonce, setRefreshNonce] = useState(0); + const [focusedRowId, setFocusedRowId] = useState(null); + const [cleanupConfirm, setCleanupConfirm] = useState<{ retentionDays: number } | null>(null); + const [cleanupBusy, setCleanupBusy] = useState(false); + const [cleanupOutcome, setCleanupOutcome] = useState(null); + const [deleteTarget, setDeleteTarget] = useState(null); + const [deleteBusy, setDeleteBusy] = useState(false); + const [deleteError, setDeleteError] = useState(null); + const [restoreTarget, setRestoreTarget] = useState(null); + const generation = useRef(0); + const emptyPageFallbackUsed = useRef(false); + + const dateRangeError = validateHistoryDateRange( + inventory.filters.createdFrom, + inventory.filters.createdTo, + ); + const selected = page.backups.find((backup) => backup.id === inventory.selectedId) ?? null; + const hasPrevious = inventory.pager.previous.length > 0; + const hasNext = Boolean(page.pagination.has_more && inventory.pager.next); + + const setFilters = (update: Partial) => { + emptyPageFallbackUsed.current = false; + updateInventory((state) => ({ + ...state, + filters: { ...state.filters, ...update }, + pager: FIRST_PAGE, + selectedId: null, + })); + setFocusedRowId(null); + setCleanupOutcome(null); + }; + + const selectBackup = (backupId: string) => { + updateInventory({ selectedId: backupId }); + requestAnimationFrame(() => { + if (isStacked(listRef.current, detailRef.current)) { + detailRef.current?.scrollIntoView({ block: 'start' }); + } + }); + }; + + const onListKeyDown = (event: KeyboardEvent) => { + if (!['ArrowDown', 'ArrowUp', 'Home', 'End'].includes(event.key)) return; + const buttons = Array.from( + event.currentTarget.querySelectorAll('[data-backup-row]'), + ); + if (!buttons.length) return; + event.preventDefault(); + const index = buttons.indexOf(document.activeElement as HTMLButtonElement); + const next = + event.key === 'Home' + ? 0 + : event.key === 'End' + ? buttons.length - 1 + : event.key === 'ArrowDown' + ? Math.min(buttons.length - 1, index + 1) + : Math.max(0, index - 1); + buttons[next]?.focus(); + }; + + useEffect(() => { + if (!visible || dateRangeError) return undefined; + const controller = new AbortController(); + const requestGeneration = generation.current + 1; + generation.current = requestGeneration; + setLoading(true); + setFailure(null); + const filters = inventory.filters; + const token = inventory.pager.current; + + listBackups(filters, token, controller.signal) + .then((nextPage) => { + if (controller.signal.aborted || generation.current !== requestGeneration) return; + if ( + !emptyPageFallbackUsed.current && + token !== null && + inventory.pager.previous.length > 0 && + nextPage.backups.length === 0 && + !nextPage.pagination.next_token + ) { + emptyPageFallbackUsed.current = true; + updateInventory((state) => + state.pager.current === token + ? { ...state, pager: pagerBack(state.pager), selectedId: null } + : state, + ); + return; + } + if (token === null || nextPage.backups.length || nextPage.pagination.next_token) { + emptyPageFallbackUsed.current = false; + } + setPage(nextPage); + updateInventory((state) => ({ + ...state, + pager: + state.pager.current === token + ? pagerAfterLoad(state.pager, nextPage.pagination.next_token) + : state.pager, + selectedId: + state.selectedId && nextPage.backups.some((backup) => backup.id === state.selectedId) + ? state.selectedId + : null, + })); + }) + .catch((error: unknown) => { + if (controller.signal.aborted || generation.current !== requestGeneration) return; + setFailure(readHistoryFailure(error, 'Backup inventory could not be loaded.')); + }) + .finally(() => { + if (!controller.signal.aborted && generation.current === requestGeneration) setLoading(false); + }); + + return () => controller.abort(); + }, [ + visible, + dateRangeError, + inventory.filters.status, + inventory.filters.scheduled, + inventory.filters.pageSize, + inventory.filters.backupType, + inventory.filters.createdFrom, + inventory.filters.createdTo, + inventory.pager.current, + backupsRevision, + refreshNonce, + updateInventory, + ]); + + const tabStopId = + (focusedRowId && page.backups.some((backup) => backup.id === focusedRowId) && focusedRowId) || + (inventory.selectedId && + page.backups.some((backup) => backup.id === inventory.selectedId) && + inventory.selectedId) || + page.backups[0]?.id; + + const summary: BackupGlobalSummary = page.summary ?? {}; + + const runCleanup = async () => { + const epoch = currentEpoch(); + setCleanupBusy(true); + setCleanupOutcome(null); + try { + const result = await useDataManagementStore.getState().trackRequest(runRetentionCleanup()); + if (!isCurrentEpoch(epoch)) return; + setCleanupOutcome(result); + useDataManagementStore.getState().notifyBackupsChanged(); + useDataManagementStore.getState().notifyJobsChanged(); + if (count(result.errors?.length)) { + toast.error('Retention cleanup completed with errors.'); + } else { + toast.success(cleanupResultMessage(result)); + } + setCleanupConfirm(null); + } catch (error) { + if (!isCurrentEpoch(epoch)) return; + toast.error(errorMessage(error, 'Backup retention cleanup could not be completed.')); + } finally { + if (isCurrentEpoch(epoch)) setCleanupBusy(false); + } + }; + + const confirmCleanup = async () => { + const epoch = currentEpoch(); + if (!(await ensure('retention-cleanup')) || !isCurrentEpoch(epoch)) return; + const state = useDataManagementStore.getState(); + const freshValues = readDmValues(state.settings, state.draft); + setCleanupConfirm({ + retentionDays: computeRetentionDays(freshValues.retention_value, freshValues.retention_unit), + }); + }; + + const runDelete = async () => { + if (!deleteTarget?.id) return; + const epoch = currentEpoch(); + setDeleteBusy(true); + setDeleteError(null); + try { + const result = await useDataManagementStore + .getState() + .trackRequest(deleteBackup(deleteTarget.id)); + if (!isCurrentEpoch(epoch)) return; + toast.success( + `Deleted ${formatBackupType(deleteTarget.backup_type)} backup and ${formatNumber(result.deleted_blob_count)} stored artifact${count(result.deleted_blob_count) === 1 ? '' : 's'}.`, + ); + const state = useDataManagementStore.getState(); + state.notifyBackupsChanged(); + state.notifyJobsChanged(); + if (state.jobs.selectedId === deleteTarget.id) { + state.updateJobs({ selectedId: null }); + } + updateInventory({ selectedId: null }); + setDeleteTarget(null); + } catch (error) { + if (!isCurrentEpoch(epoch)) return; + setDeleteError(errorMessage(error, 'Backup could not be deleted.')); + } finally { + if (isCurrentEpoch(epoch)) setDeleteBusy(false); + } + }; + + const list = ( +
+ {loading && !page.backups.length ? ( +
+ + + +
+ ) : page.backups.length ? ( +
    + {page.backups.map((backup) => { + const isSelected = backup.id === inventory.selectedId; + const warnings = count(backup.warning_count); + const metaId = `dm-backup-row-${backup.id}`; + return ( +
  • + +
    • + ); + })} +
+ ) : ( + + {visible + ? 'Change the filters or refresh after a backup job finishes.' + : 'The list loads when this card is visible.'} + + )} +
+ ); + + const detail = selected ? ( +
+
+
+

+ {formatBackupType(selected.backup_type)} backup +

+

{selected.id}

+
+
+ + {runType(selected)} +
+
+ + {selected.id} }, + { label: 'Type', value: formatBackupType(selected.backup_type) }, + { label: 'Status', value: }, + { label: 'Run type', value: runType(selected) }, + { label: 'Created', value: formatDateTime(selected.created_at) || 'Not recorded' }, + { label: 'Completed', value: formatDateTime(selected.completed_at) || 'Not recorded' }, + { label: 'Contents', value: contentsLabel(selected) }, + { label: 'Size', value: formatBytes(selected.bytes) }, + { label: 'Artifacts', value: formatNumber(selected.artifact_count) }, + { label: 'Protection', value: selected.encrypted ? 'Encrypted' : 'Not encrypted' }, + { label: 'Warnings', value: warningLabel(selected) }, + { label: 'Manifest', value: selected.manifest_path ? 'Recorded' : 'Missing' }, + { + label: 'Storage prefix', + value: ( + {text(selected.base_prefix, 'Not recorded')} + ), + }, + ]} + /> + + {selected.last_message ? {selected.last_message} : null} + +
+ setRestoreTarget(selected)} + > + + {selected.can_delete ? ( + { + setDeleteError(null); + setDeleteTarget(selected); + }} + > + + ) : null} + { + useDataManagementStore.getState().focusJob(selected.id); + onNavigate(DM_SECTION_IDS.jobs); + }} + > + +
+ + {!canRestore(selected) ? ( +

+ Restore is available only for completed backups with a recorded manifest. +

+ ) : null} +
+ ) : ( + + Choose a backup row to inspect its manifest, storage and restore actions. + + ); + + return ( +
+
+ + {help ?? + 'Track completed full and partial backups, review their contents, and restore or delete them deliberately.'} + +
+ void confirmCleanup()} + > + + setRefreshNonce((value) => value + 1)} + > + +
+
+ +
+
+ +
+ setFilters({ status })} + /> + + setFilters({ scheduled: scheduled as typeof inventory.filters.scheduled }) + } + /> + setFilters({ createdFrom })} + /> + setFilters({ createdTo })} + /> + setFilters({ pageSize: Number(pageSize) })} + /> +
+ + {dateRangeError ? ( + + {dateRangeError} + + ) : null} + + {failure ? ( + setRefreshNonce((value) => value + 1)} + > + Retry + + ) : null + } + > + {failure.message} + + ) : null} + + {cleanupOutcome ? ( + + + {cleanupResultMessage(cleanupOutcome)} Candidates:{' '} + {formatNumber(cleanupOutcome.candidate_count)}. Errors:{' '} + {formatNumber(cleanupOutcome.errors?.length)}. Cutoff:{' '} + {formatDateTime(cleanupOutcome.cutoff_at) || 'not returned'}. + + + ) : null} + + + + { + emptyPageFallbackUsed.current = false; + updateInventory((state) => ({ + ...state, + pager: pagerBack(state.pager), + selectedId: null, + })); + }} + onNext={() => { + emptyPageFallbackUsed.current = false; + updateInventory((state) => ({ + ...state, + pager: pagerForward(state.pager), + selectedId: null, + })); + }} + /> + + {cleanupConfirm ? ( + + ) : null} + + {deleteTarget ? ( + + ) : null} + + {restoreTarget ? ( + setRestoreTarget(null)} + onNavigate={onNavigate} + /> + ) : null} + + {dialog} +
+ ); +} diff --git a/application/v2_ui/src/components/admin/dataManagement/DataManagementBackupRuns.tsx b/application/v2_ui/src/components/admin/dataManagement/DataManagementBackupRuns.tsx new file mode 100644 index 000000000..db56bfa38 --- /dev/null +++ b/application/v2_ui/src/components/admin/dataManagement/DataManagementBackupRuns.tsx @@ -0,0 +1,324 @@ +// DataManagementBackupRuns.tsx +// Backup: queue a backup now and tune backup performance. + +import { useEffect, useMemo, useRef, useState } from 'react'; +import { Archive, ArchiveRestore, ExternalLink } from 'lucide-react'; +import { ApiError } from '../../../lib/apiClient'; +import { queueJob, type BackupRow, type BackupType } from '../../../lib/dataManagement'; +import { + asFlag, + blobBufferEstimateMib, + DM_SECTION_IDS, + formatBackupType, + formatDateTime, + isBackupStorageConfigured, + readDmValues, +} from '../../../lib/dataManagementLogic'; +import { + currentEpoch, + isCurrentEpoch, + useDataManagementStore, + useDmDirtyCount, + useDmValues, +} from '../../../stores/dataManagementStore'; +import { toast } from '../../../stores/toastStore'; +import { GlassButton, Skeleton } from '../../ui/primitives'; +import { DmField, useDmValue } from './DmField'; +import type { DmCardProps } from './DmShared'; +import { DmDisclosure, DmIntro, DmNotice, DmStatusPill, useVisibleOnce } from './DmShared'; +import { DataManagementGuideDialog } from './DataManagementGuides'; +import { useBackupSummary } from './useBackupSummary'; +import { useSaveFirst } from './useSaveFirst'; + +interface QueueResult { + jobId: string; + backupType: BackupType; +} + +function backupLine(label: string, backup: BackupRow | null | undefined) { + if (!backup) { + return ( +
+
{label}
+
None yet
+
+ ); + } + const completedAt = backup.completed_at || backup.created_at; + return ( +
+
{label}
+
+ + completed {formatDateTime(completedAt)} +
+
+ ); +} + +export function DataManagementBackupRuns({ help, onNavigate, disabled }: DmCardProps) { + const rootRef = useRef(null); + const visible = useVisibleOnce(rootRef); + const values = useDmValues(); + const dirtyCount = useDmDirtyCount(); + const sourceRuEnabled = asFlag(useDmValue('backup_temporary_source_ru_enabled')); + const storageConfigured = isBackupStorageConfigured(values); + const { summary, loading, failure } = useBackupSummary(visible); + const { ensure, dialog } = useSaveFirst(); + const [queueing, setQueueing] = useState(null); + const [result, setResult] = useState(null); + const [actionError, setActionError] = useState(''); + const [guideOpen, setGuideOpen] = useState(false); + + useEffect(() => { + void useDataManagementStore.getState().ensureLoaded(); + }, []); + + const bufferEstimate = useMemo(() => blobBufferEstimateMib(values), [values]); + const queueDisabled = Boolean(disabled || queueing || !storageConfigured); + + const queueBackup = async (backupType: BackupType) => { + setActionError(''); + setResult(null); + if (!(await ensure('queue-backup'))) return; + + const state = useDataManagementStore.getState(); + const freshValues = readDmValues(state.settings, state.draft); + const options = { + include_cosmos: Boolean(freshValues.include_cosmos), + include_ai_search: Boolean(freshValues.include_ai_search), + include_source_blobs: Boolean(freshValues.include_source_blobs), + }; + + const token = currentEpoch(); + setQueueing(backupType); + try { + const job = await state.trackRequest(queueJob('backup', backupType, options)); + if (!isCurrentEpoch(token)) return; + toast.success(`${formatBackupType(backupType)} backup queued.`); + useDataManagementStore.getState().notifyJobsChanged(); + useDataManagementStore.getState().notifyBackupsChanged(); + setResult({ jobId: job.id, backupType }); + } catch (error: unknown) { + if (!isCurrentEpoch(token)) return; + if (error instanceof ApiError && error.status >= 400 && error.status < 500) { + setActionError(error.message || 'Backup could not be queued.'); + return; + } + setActionError( + 'The request may not have reached the server. Check Job history before queueing another backup.', + ); + useDataManagementStore.getState().notifyJobsChanged(); + } finally { + if (isCurrentEpoch(token)) setQueueing(null); + } + }; + + return ( +
+ + {help ?? + 'Queue an immediate backup with the saved Backup & Recovery settings, then tune source Cosmos and source file transfer performance.'} + + + {!storageConfigured ? ( + onNavigate(DM_SECTION_IDS.storage)} + > + Open storage + + } + > + Configure a dedicated backup storage account before queueing full or partial backups. + + ) : null} + +
+
+
+

+ Run a backup now +

+

+ Queue an immediate full or partial backup. Jobs use Cosmos-backed leases so + scaled-out App Service instances do not run the same backup twice. +

+ {dirtyCount > 0 ? ( +

+ Pending Backup & Recovery settings are saved first. +

+ ) : null} +
+
+ void queueBackup('full')} + > + + void queueBackup('partial')} + > + +
+
+ + {queueing ? ( +

+ Queueing {formatBackupType(queueing).toLowerCase()} backup… +

+ ) : null} + + {actionError ? ( + + {actionError} + + ) : null} + + {result ? ( + { + useDataManagementStore.getState().focusJob(result.jobId); + onNavigate(DM_SECTION_IDS.jobs); + }} + > + + ) : null} +
+ +
+
+

+ Latest results +

+ {loading ? ( + + Loading latest backups… + + ) : null} +
+ {loading && !summary ? ( +
+ + +
+ ) : ( +
+ {backupLine('Latest full backup', summary?.latest_full)} + {backupLine('Latest partial backup', summary?.latest_partial)} +
+ )} + {failure ? ( + + {failure.message} + + ) : null} +
+ + +
+

+ Backups stream checkpointed Cosmos batches and commit verified work. Higher + concurrency can finish sooner and can also increase source Cosmos pressure. +

+
+ + + +
+ + setGuideOpen(true)} + > + About RU Boost + +
+ } + /> + {sourceRuEnabled ? ( + + ) : null} +
+ + + +
+

+ Source files stream through bounded chunks and durable per-file checkpoints. +

+
+ + + +
+

+ About {bufferEstimate} MiB of file data can be buffered at once at most, excluding + Azure SDK overhead. Throttling temporarily lowers active transfers. +

+
+ + + {guideOpen ? ( + setGuideOpen(false)} /> + ) : null} + {dialog} + + ); +} diff --git a/application/v2_ui/src/components/admin/dataManagement/DataManagementConfigParts.tsx b/application/v2_ui/src/components/admin/dataManagement/DataManagementConfigParts.tsx new file mode 100644 index 000000000..ddbd4b44e --- /dev/null +++ b/application/v2_ui/src/components/admin/dataManagement/DataManagementConfigParts.tsx @@ -0,0 +1,109 @@ +// DataManagementConfigParts.tsx +// Shared card chrome keeps the three backup configuration cards aligned without moving settings ownership into foundation files. + +import { useEffect, useMemo, type ReactNode } from 'react'; +import { RefreshCw } from 'lucide-react'; +import { type DmEditableKey, type DmSettings } from '../../../lib/dataManagement'; +import { dmDirtyKeys } from '../../../lib/dataManagementLogic'; +import { useDataManagementStore } from '../../../stores/dataManagementStore'; +import { GlassButton, Skeleton } from '../../ui/primitives'; +import { DmNotice } from './DmShared'; + +export function useDmConfigCard(ownerKeys: readonly DmEditableKey[], fallbackForSaveError = false) { + const status = useDataManagementStore((state) => state.status); + const loadError = useDataManagementStore((state) => state.loadError); + const settings = useDataManagementStore((state) => state.settings); + const draft = useDataManagementStore((state) => state.draft); + const saveError = useDataManagementStore((state) => state.saveError); + const saving = useDataManagementStore((state) => state.saving); + + useEffect(() => { + void useDataManagementStore.getState().ensureLoaded(); + }, []); + + const ownedSaveError = useMemo(() => { + if (!saveError) return null; + const dirty = dmDirtyKeys(settings, draft); + const ownsDirtyKey = dirty.some((key) => ownerKeys.includes(key)); + if (ownsDirtyKey || (fallbackForSaveError && dirty.length === 0)) { + return saveError; + } + return null; + }, [draft, fallbackForSaveError, ownerKeys, saveError, settings]); + + return { + status, + loadError, + settings, + saving, + saveError: ownedSaveError, + }; +} + +export function DmLoadingRows({ rows = 5 }: { rows?: number }) { + return ( +
+ {Array.from({ length: rows }).map((_, index) => ( +
+ + +
+ ))} +
+ ); +} + +export function DmLoadError({ message }: { message: string | null }) { + return ( + void useDataManagementStore.getState().reload()} + > + + ); +} + +export function DmCardSaveError({ message }: { message: string | null }) { + if (!message) return null; + return ( + + {message} + + ); +} + +export function DmDefinitionReadout({ + label, + value, + children, +}: { + label: string; + value: ReactNode; + children?: ReactNode; +}) { + return ( +
+
{label}
+
{value}
+ {children ?
{children}
: null} +
+ ); +} + +export function hasGeneratedKey(settings: DmSettings | null | undefined): boolean { + const reference = settings?.encryption_key_reference; + return typeof reference === 'string' ? reference.trim().length > 0 : Boolean(reference); +} diff --git a/application/v2_ui/src/components/admin/dataManagement/DataManagementCosmosEditor.tsx b/application/v2_ui/src/components/admin/dataManagement/DataManagementCosmosEditor.tsx new file mode 100644 index 000000000..b4842fd78 --- /dev/null +++ b/application/v2_ui/src/components/admin/dataManagement/DataManagementCosmosEditor.tsx @@ -0,0 +1,1054 @@ +// DataManagementCosmosEditor.tsx +// The editor keeps its unlocked state and document draft in the shared store so risky +// Cosmos work survives card unmounts but is still discarded when the Admin page closes. + +import { useCallback, useEffect, useId, useMemo, useRef, useState, type KeyboardEvent } from 'react'; +import { clsx } from 'clsx'; +import { DatabaseZap, FileJson, Loader2, Lock, RotateCw, Save, Search, Unlock } from 'lucide-react'; +import { ApiError } from '../../../lib/apiClient'; +import { + COSMOS_EDITOR_MAX_PAGE_SIZE, + COSMOS_EDITOR_SAVE_PHRASE, + acknowledgeCosmosDanger, + errorMessage, + listCosmosContainers, + openCosmosDocument, + queryCosmos, + saveCosmosDocument, + type CosmosContainer, + type CosmosDocumentResult, + type CosmosQueryItem, +} from '../../../lib/dataManagement'; +import { + checkCosmosEdit, + formatCosmosDocument, + formatDetailValue, + formatNumber, + summarizeCosmosChanges, + validateCosmosQuery, +} from '../../../lib/dataManagementLogic'; +import { + currentEpoch, + isCurrentEpoch, + selectCosmosEditorDirty, + useDataManagementStore, + type CosmosOpenDocument, +} from '../../../stores/dataManagementStore'; +import { toast } from '../../../stores/toastStore'; +import { ConfirmDialog } from '../../ui/ConfirmDialog'; +import { Modal } from '../../ui/Modal'; +import { GlassButton } from '../../ui/primitives'; +import { inputClass } from '../fields'; +import { + DmEmpty, + DmIntro, + DmMetricGrid, + DmNotice, + DmPhraseField, + DmWorkbench, + isStacked, + type DmCardProps, +} from './DmShared'; + +type BusyAction = 'unlock' | 'containers' | 'query' | 'next' | 'open' | 'reload' | 'save' | null; + +interface PendingConfirm { + title: string; + description: string; + confirmLabel: string; + onConfirm: () => void; +} + +function labelForContainer(container: CosmosContainer): string { + const name = container.display_name || container.name || container.id; + return `${name} (${container.partition_key_path || 'no partition key path'})${container.editable === false ? ' (read-only)' : ''}`; +} + +function categoryForContainer(container: CosmosContainer): string { + return container.category || 'Other'; +} + +function groupedContainers( + containers: CosmosContainer[], +): Array<{ category: string; containers: CosmosContainer[] }> { + const groups = new Map(); + for (const container of [...containers].sort((left, right) => { + const category = categoryForContainer(left).localeCompare(categoryForContainer(right)); + return category || labelForContainer(left).localeCompare(labelForContainer(right)); + })) { + const category = categoryForContainer(container); + groups.set(category, [...(groups.get(category) ?? []), container]); + } + return [...groups.entries()].map(([category, items]) => ({ category, containers: items })); +} + +function resultPreview(item: CosmosQueryItem): string { + const preview = item.preview?.trim() || 'No preview fields available.'; + return preview.length > 120 ? `${preview.slice(0, 117)}…` : preview; +} + +function documentFromResult( + result: CosmosDocumentResult, + fallback: CosmosOpenDocument | null, +): CosmosOpenDocument { + const document = + result.document && typeof result.document === 'object' && !Array.isArray(result.document) + ? result.document + : {}; + return { + container: result.container?.name || fallback?.container || '', + id: result.id, + partitionKey: result.partition_key, + partitionKeyPath: result.container?.partition_key_path || fallback?.partitionKeyPath || '', + etag: result.etag ?? null, + original: document, + text: formatCosmosDocument(document), + editable: result.container?.editable ?? fallback?.editable ?? true, + }; +} + +function queryStatus( + result: { items: CosmosQueryItem[]; query?: { mode?: 'empty' | 'custom' }; has_more?: boolean }, + total: number, +): string { + const source = result.query?.mode === 'empty' ? 'a browse' : 'a custom SELECT query'; + return `${formatNumber(total)} loaded from ${source}. ${result.has_more ? 'More results are available.' : 'No more results.'}`; +} + +function serverSummary(result: CosmosDocumentResult): string | null { + if (!result.change_summary) return null; + const count = formatNumber( + result.change_summary.changed_count ?? result.change_summary.changed_paths?.length ?? 0, + ); + return `Server recorded ${count} changed path${count === '1' ? '' : 's'} (${formatNumber(result.change_summary.added_count ?? 0)} added, ${formatNumber(result.change_summary.removed_count ?? 0)} removed, ${formatNumber(result.change_summary.updated_count ?? 0)} updated).`; +} + +export function DataManagementCosmosEditor({ help, disabled }: DmCardProps) { + const baseId = useId(); + const cosmos = useDataManagementStore((state) => state.cosmos); + const dirty = useDataManagementStore(selectCosmosEditorDirty); + const [dangerOpen, setDangerOpen] = useState(false); + const [dangerAccepted, setDangerAccepted] = useState(false); + const [busy, setBusy] = useState(null); + const [notice, setNotice] = useState(null); + const [error, setError] = useState(null); + const [pendingConfirm, setPendingConfirm] = useState(null); + const [saveOpen, setSaveOpen] = useState(false); + const [savePhrase, setSavePhrase] = useState(''); + const [saveError, setSaveError] = useState(null); + const [lastSaveSummary, setLastSaveSummary] = useState(null); + const listRef = useRef(null); + const detailRef = useRef(null); + + const selectedContainer = useMemo( + () => + cosmos.containers.find( + (container) => container.name === cosmos.container || container.id === cosmos.container, + ) ?? null, + [cosmos.container, cosmos.containers], + ); + const containerGroups = useMemo(() => groupedContainers(cosmos.containers), [cosmos.containers]); + const queryError = validateCosmosQuery(cosmos.query); + const editCheck = cosmos.document + ? checkCosmosEdit(cosmos.document.text, { + id: cosmos.document.id, + partitionKey: cosmos.document.partitionKey, + partitionKeyPath: cosmos.document.partitionKeyPath, + }) + : null; + const parsedDocument = editCheck?.ok ? editCheck.document : null; + const localSummary = + parsedDocument && cosmos.document + ? summarizeCosmosChanges(cosmos.document.original, parsedDocument) + : null; + const controlsDisabled = Boolean(disabled || busy); + const canRunQuery = cosmos.unlocked && Boolean(cosmos.container) && !queryError && !controlsDisabled; + const canSave = Boolean(cosmos.document?.editable && editCheck?.ok && !controlsDisabled); + + useEffect(() => { + if (!dirty) return; + const onBeforeUnload = (event: BeforeUnloadEvent) => { + event.preventDefault(); + event.returnValue = ''; + }; + window.addEventListener('beforeunload', onBeforeUnload); + return () => window.removeEventListener('beforeunload', onBeforeUnload); + }, [dirty]); + + const askBeforeDiscard = useCallback( + (action: () => void, title: string, description: string, confirmLabel: string) => { + if (!dirty) { + action(); + return; + } + setPendingConfirm({ title, description, confirmLabel, onConfirm: action }); + }, + [dirty], + ); + + const loadContainers = useCallback(async () => { + const token = currentEpoch(); + setBusy('containers'); + setError(null); + try { + const containers = await listCosmosContainers(); + if (!isCurrentEpoch(token)) return; + useDataManagementStore.getState().updateCosmos((current) => ({ + ...current, + containers, + container: current.container || containers[0]?.name || '', + })); + if (!containers.length) setNotice('No Cosmos DB containers were returned.'); + } catch (failure) { + if (!isCurrentEpoch(token)) return; + setError(errorMessage(failure, 'Cosmos DB containers could not be loaded.')); + useDataManagementStore.getState().updateCosmos({ containers: [], container: '' }); + } finally { + if (isCurrentEpoch(token)) setBusy(null); + } + }, []); + + // The list loads once when the editor is unlocked. A failed or empty load waits for + // Retry rather than reloading on its own, so an outage is not hammered with requests. + + const unlock = async () => { + if (!dangerAccepted) return; + const token = currentEpoch(); + setBusy('unlock'); + setError(null); + try { + await useDataManagementStore.getState().trackRequest(acknowledgeCosmosDanger()); + if (!isCurrentEpoch(token)) return; + useDataManagementStore.getState().updateCosmos({ unlocked: true }); + setDangerOpen(false); + setDangerAccepted(false); + setNotice( + 'Cosmos DB editor unlocked for this page session. The acknowledgement was recorded in activity logs.', + ); + await loadContainers(); + } catch (failure) { + if (!isCurrentEpoch(token)) return; + setError(errorMessage(failure, 'Cosmos DB editor could not be unlocked.')); + } finally { + if (isCurrentEpoch(token)) setBusy(null); + } + }; + + const lockEditor = () => { + askBeforeDiscard( + () => { + useDataManagementStore.getState().lockCosmos(); + setNotice('Cosmos DB editor locked.'); + setError(null); + setLastSaveSummary(null); + }, + 'Discard the open JSON draft?', + 'Locking the editor clears the open document and any unsaved JSON changes in this page session.', + 'Discard and lock', + ); + }; + + const selectContainer = (container: string) => { + askBeforeDiscard( + () => { + useDataManagementStore.getState().updateCosmos({ + container, + results: [], + continuationToken: null, + queryMode: null, + queryStatus: 'No query has run yet.', + document: null, + }); + setLastSaveSummary(null); + }, + 'Discard the open JSON draft?', + 'Changing containers closes the current document and clears unsaved JSON changes.', + 'Discard and change container', + ); + }; + + const runQuery = async (useContinuation: boolean) => { + if ( + !cosmos.unlocked || + !cosmos.container || + (useContinuation && !cosmos.continuationToken) || + queryError + ) + return; + const execute = async () => { + const token = currentEpoch(); + setBusy(useContinuation ? 'next' : 'query'); + setError(null); + setLastSaveSummary(null); + useDataManagementStore.getState().updateCosmos({ + queryStatus: useContinuation ? 'Loading next page…' : 'Running query…', + ...(useContinuation ? {} : { document: null, results: [], continuationToken: null }), + }); + try { + const result = await queryCosmos( + cosmos.container, + cosmos.query, + cosmos.pageSize, + useContinuation ? cosmos.continuationToken : null, + ); + if (!isCurrentEpoch(token)) return; + const previous = useContinuation ? useDataManagementStore.getState().cosmos.results : []; + const results = [...previous, ...(result.items ?? [])]; + useDataManagementStore.getState().updateCosmos({ + results, + continuationToken: result.continuation_token ?? null, + queryMode: result.query?.mode ?? (cosmos.query.trim() ? 'custom' : 'empty'), + queryStatus: queryStatus(result, results.length), + }); + if (!result.items.length && !useContinuation) setNotice('No documents matched this query.'); + } catch (failure) { + if (!isCurrentEpoch(token)) return; + const message = errorMessage(failure, 'Cosmos DB query failed.'); + setError(message); + useDataManagementStore.getState().updateCosmos({ queryStatus: message }); + } finally { + if (isCurrentEpoch(token)) setBusy(null); + } + }; + if (useContinuation) { + await execute(); + } else { + askBeforeDiscard( + () => { + void execute(); + }, + 'Discard the open JSON draft?', + 'Running a new query closes the current document and clears unsaved JSON changes.', + 'Discard and run query', + ); + } + }; + + const openDocument = async (item: CosmosQueryItem) => { + if (!item.id || item.selectable !== true) return; + const execute = async () => { + const token = currentEpoch(); + setBusy('open'); + setError(null); + setLastSaveSummary(null); + try { + const result = await openCosmosDocument( + cosmos.container, + item.id as string, + item.partition_key, + ); + if (!isCurrentEpoch(token)) return; + useDataManagementStore + .getState() + .updateCosmos({ document: documentFromResult(result, null) }); + window.setTimeout(() => { + if (isStacked(listRef.current, detailRef.current)) { + detailRef.current?.scrollIntoView({ block: 'start' }); + } else { + detailRef.current?.focus({ preventScroll: true }); + } + }, 0); + } catch (failure) { + if (!isCurrentEpoch(token)) return; + setError(errorMessage(failure, 'Cosmos DB document could not be opened.')); + useDataManagementStore.getState().updateCosmos({ document: null }); + } finally { + if (isCurrentEpoch(token)) setBusy(null); + } + }; + askBeforeDiscard( + () => { + void execute(); + }, + 'Discard the open JSON draft?', + 'Opening another document clears the unsaved JSON changes in the current document.', + 'Discard and open document', + ); + }; + + const reloadDocumentNow = async () => { + const document = cosmos.document; + if (!document) return; + const token = currentEpoch(); + setBusy('reload'); + setError(null); + setSaveError(null); + setLastSaveSummary(null); + try { + const result = await openCosmosDocument(document.container, document.id, document.partitionKey); + if (!isCurrentEpoch(token)) return; + useDataManagementStore + .getState() + .updateCosmos({ document: documentFromResult(result, document) }); + setNotice('Document reloaded from Cosmos DB.'); + } catch (failure) { + if (!isCurrentEpoch(token)) return; + setError(errorMessage(failure, 'Cosmos DB document could not be reloaded.')); + } finally { + if (isCurrentEpoch(token)) setBusy(null); + } + }; + + const reloadDocument = async () => { + askBeforeDiscard( + () => { + void reloadDocumentNow(); + }, + 'Discard unsaved JSON changes?', + 'Reloading reads the current document from Cosmos DB and replaces the JSON in the editor.', + 'Discard and reload', + ); + }; + + const resetChanges = () => { + const document = cosmos.document; + if (!document) return; + useDataManagementStore.getState().updateCosmos({ + document: { ...document, text: formatCosmosDocument(document.original) }, + }); + setSaveError(null); + setLastSaveSummary(null); + }; + + const saveDocument = async () => { + const document = cosmos.document; + if (!document || !parsedDocument || !document.etag) return; + const token = currentEpoch(); + setBusy('save'); + setSaveError(null); + try { + const result = await useDataManagementStore.getState().trackRequest( + saveCosmosDocument({ + container: document.container, + id: document.id, + partitionKey: document.partitionKey, + etag: document.etag, + document: parsedDocument, + confirmationPhrase: savePhrase, + }), + ); + if (!isCurrentEpoch(token)) return; + useDataManagementStore + .getState() + .updateCosmos({ document: documentFromResult(result, document) }); + setLastSaveSummary(serverSummary(result)); + setSaveOpen(false); + toast.success('Cosmos DB document saved. The edit was recorded in activity logs.'); + } catch (failure) { + if (!isCurrentEpoch(token)) return; + if (failure instanceof ApiError && failure.status === 409) { + setSaveError( + 'This document changed after it was opened. Reload it, reapply the edit, then save again.', + ); + } else { + setSaveError(errorMessage(failure, 'Cosmos DB document could not be saved.')); + } + } finally { + setSavePhrase(''); + if (isCurrentEpoch(token)) setBusy(null); + } + }; + + const handleListKeyDown = (event: KeyboardEvent) => { + if (event.key !== 'ArrowDown' && event.key !== 'ArrowUp') return; + const buttons = [ + ...(listRef.current?.querySelectorAll('button[data-result-row="true"]') ?? []), + ].filter((button) => !button.disabled); + if (!buttons.length) return; + const current = + document.activeElement instanceof HTMLButtonElement + ? buttons.indexOf(document.activeElement) + : -1; + const next = + event.key === 'ArrowDown' + ? (current + 1) % buttons.length + : current <= 0 + ? buttons.length - 1 + : current - 1; + event.preventDefault(); + buttons[next]?.focus(); + }; + + return ( +
+ + {help || + 'Query a known SimpleChat Cosmos DB container, inspect one document, and save guarded JSON repairs with ETag protection.'} + +
+ + This tool changes production Cosmos DB documents directly. Use it only for targeted repair + or investigation after the impact is understood. + + {notice ? ( + + {notice} + + ) : null} + {error ? ( + + {error} + + ) : null} + {!cosmos.unlocked ? ( +
+
+

The Cosmos DB JSON editor is locked.

+

+ Unlock it only after you know which container and document you need. Unlocking + records an acknowledgement in activity logs and enables live queries for this + page session. +

+
+ { + setDangerAccepted(false); + setDangerOpen(true); + }} + > + +
+ ) : ( +
+
+

+ Editor unlocked for this page session. Queries and saves are recorded by the + server. +

+ + +
+ {!cosmos.containers.length && busy !== 'containers' ? ( + void loadContainers()} + > + + ) : null} +
+
+ + +

+ {selectedContainer + ? `${categoryForContainer(selectedContainer)} · ${selectedContainer.partition_key_path || 'No partition key path'}` + : 'Category · partition key'} +

+
+
+ + { + const next = Math.max( + 1, + Math.min( + COSMOS_EDITOR_MAX_PAGE_SIZE, + Number.parseInt(event.target.value, 10) || + COSMOS_EDITOR_MAX_PAGE_SIZE, + ), + ); + // A continuation token belongs to the query and page size that produced it. + useDataManagementStore + .getState() + .updateCosmos({ pageSize: next, continuationToken: null }); + }} + /> +

+ 1 to {COSMOS_EDITOR_MAX_PAGE_SIZE}. +

+
+
+ +