From 7a8b301220228c5a2b475656884a499cc32ac25f Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 6 Oct 2026 22:21:11 -0400 Subject: [PATCH] Bring V2 admin Data Lifecycle settings to V1 parity The React V2 Admin Settings drew Retention Policy, Document Classification and Conversation Archiving through the enable_* fallback, as bare switches with their storage keys printed beneath them. Describe all three sections in the V2 field schema and add the controls V1 offers: - Per-type retention switches with nested conversation/document defaults, a run hour select that saves an int, and a last/next run readout - Inline Run now and Reset to defaults reviews, limited to types switched on in saved settings and held while edits are unsaved; results show per-type counts and never raw server exception text - A document classification categories editor with validation and badge preview - Reschedule the next retention run when a V2 save changes a scope switch or the run hour, matching the classic save handler Make public workspace conversation retention functional. Chats grounded in a public workspace follow that workspace's policy while it exists and return to their owner's personal policy once it is deleted; a failed workspace listing skips grounded chats for that run. Workspace managers get a count, and each owner is told which of their own chats were removed. Add parity, logic and Playwright coverage, admin/feature/fix docs, and bump the version to 0.261.260. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 416 +++++++++- application/single_app/config.py | 2 +- .../single_app/functions_retention_policy.py | 240 +++++- .../admin/ClassificationCategoriesEditor.tsx | 210 ++++++ .../components/admin/RetentionOperations.tsx | 709 ++++++++++++++++++ .../components/admin/RetentionSchedule.tsx | 191 +++++ .../admin/agentSectionAppearance.ts | 15 +- .../components/admin/retentionScopeIcons.ts | 12 + .../v2_ui/src/lib/classificationCategories.ts | 75 ++ application/v2_ui/src/lib/retentionPolicy.ts | 406 ++++++++++ .../v2_ui/src/pages/AdminSettingsPage.tsx | 62 ++ docs/admin/data-lifecycle.md | 170 ++++- docs/explanation/features/REACT_V2_UI.md | 11 + .../V2_ADMIN_DATA_LIFECYCLE_SETTINGS.md | 150 ++++ ...IC_WORKSPACE_CONVERSATION_RETENTION_FIX.md | 132 ++++ ...tion_policy_conversation_scope_coverage.py | 516 ++++++++++++- .../test_v2_admin_capability_placement.py | 11 +- .../test_v2_admin_data_lifecycle_logic.ts | 382 ++++++++++ .../test_v2_admin_data_lifecycle_parity.py | 509 +++++++++++++ .../test_v2_admin_section_logic.ts | 22 +- ui_tests/fixtures/data_lifecycle_admin.py | 163 ++++ ui_tests/test_v2_admin_data_lifecycle.py | 318 ++++++++ 22 files changed, 4659 insertions(+), 63 deletions(-) create mode 100644 application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx create mode 100644 application/v2_ui/src/components/admin/RetentionOperations.tsx create mode 100644 application/v2_ui/src/components/admin/RetentionSchedule.tsx create mode 100644 application/v2_ui/src/components/admin/retentionScopeIcons.ts create mode 100644 application/v2_ui/src/lib/classificationCategories.ts create mode 100644 application/v2_ui/src/lib/retentionPolicy.ts create mode 100644 docs/explanation/features/V2_ADMIN_DATA_LIFECYCLE_SETTINGS.md create mode 100644 docs/explanation/fixes/PUBLIC_WORKSPACE_CONVERSATION_RETENTION_FIX.md create mode 100644 functional_tests/test_v2_admin_data_lifecycle_logic.ts create mode 100644 functional_tests/test_v2_admin_data_lifecycle_parity.py create mode 100644 ui_tests/fixtures/data_lifecycle_admin.py create mode 100644 ui_tests/test_v2_admin_data_lifecycle.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 374c9632f..ea83c84ec 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -59,8 +59,8 @@ input would otherwise save to a top-level key nothing reads. See ``_apply_nested_paths``. -The Appearance, Agents & Actions, Chat, Knowledge, Workflow, Workspaces and -Security groups are described in full. Sections with no entry here fall back to the V2 surface's +The Appearance, Agents & Actions, Chat, Data Lifecycle, Knowledge, Workflow, Workspaces +and Security groups are described in full. Sections with no entry here fall back to the V2 surface's ``enable_*`` scan, so undescribed groups keep working exactly as they did. A handful of individual fields outside those groups are also declared: that scan places a key by guessing from shared word stems, and declaring a field is the @@ -89,6 +89,7 @@ import copy import json import re +from datetime import datetime, timedelta, timezone from urllib.parse import urlparse from admin_settings_secret_utils import ( @@ -354,6 +355,68 @@ ACCESS_DENIED_MESSAGE_MAX_LENGTH = 2000 +# The periods the server-rendered Retention pane offers for each organization +# default, in its order and with its wording. Stored as strings because that is what +# the classic form posts and what resolve_retention_value reads. The Data Lifecycle +# parity test compares these with the pane option for option. +RETENTION_PERIOD_OPTIONS = [ + {"value": "none", "label": "No automatic deletion"}, + {"value": "1", "label": "1 day"}, + {"value": "2", "label": "2 days"}, + {"value": "3", "label": "3 days"}, + {"value": "4", "label": "4 days"}, + {"value": "5", "label": "5 days"}, + {"value": "6", "label": "6 days"}, + {"value": "7", "label": "7 days (1 week)"}, + {"value": "10", "label": "10 days"}, + {"value": "14", "label": "14 days (2 weeks)"}, + {"value": "21", "label": "21 days (3 weeks)"}, + {"value": "30", "label": "30 days"}, + {"value": "60", "label": "60 days"}, + {"value": "90", "label": "90 days (3 months)"}, + {"value": "180", "label": "180 days (6 months)"}, + {"value": "365", "label": "365 days (1 year)"}, + {"value": "730", "label": "730 days (2 years)"}, +] + +# Workspace types retention can be switched on for, in display order. +RETENTION_SCOPES = ("personal", "group", "public") + +# Mirrors 'retention_policy_execution_hour' in functions_settings.py. The hour is UTC +# and must be stored as an int: execute_retention_policy hands it straight to +# datetime.replace(hour=...), which raises on a string and stops every run. +RETENTION_EXECUTION_HOUR_DEFAULT = 2 +RETENTION_EXECUTION_HOUR_MIN = 0 +RETENTION_EXECUTION_HOUR_MAX = 23 + +# Saving any of these reschedules the next retention run, as the classic form does. +RETENTION_SCHEDULE_KEYS = ( + "enable_retention_policy_personal", + "enable_retention_policy_group", + "enable_retention_policy_public", + "retention_policy_execution_hour", +) + +# The schedule and the run and reset actions only mean something while at least one +# workspace type has retention on. +RETENTION_ANY_SCOPE_ENABLED = { + "any_of": [ + {"key": f"enable_retention_policy_{scope}", "equals": True} + for scope in RETENTION_SCOPES + ] +} + +# A category is shown as a badge on documents and conversations, so its label is kept +# to badge length. The classic form sets no bound; this matches an external link label. +DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH = 80 + +# Mirrors the 'document_classification_categories' default in functions_settings.py. +DOCUMENT_CLASSIFICATION_DEFAULT_CATEGORIES = [ + {"label": "None", "color": "#808080"}, + {"label": "N/A", "color": "#808080"}, + {"label": "Pending", "color": "#0000FF"}, +] + ADMIN_SETTINGS_FIELDS = { "branding-section": [ @@ -5764,6 +5827,206 @@ "depends_on": {"key": "enable_custom_rate_limit_message", "equals": True}, }, ], + # ------------------------------------------------------------------ + # Data Lifecycle + # + # What happens to user data over time: retention deletes it, classification + # labels it, and archiving decides whether a deletion can still be reviewed. + # + # Retention has no single switch. Each workspace type is switched on by itself + # and carries its own pair of organization defaults, which the server-rendered + # pane only reveals once that type is on; declaring the defaults as dependents + # of their type's switch gives V2 the same reveal and nests them beneath it. + # + # The run hour and the two actions are components. The hour is stored as an int, + # which a generic select cannot produce, and the actions call + # /api/admin/retention-policy/* rather than saving a value. + # ------------------------------------------------------------------ + "retention-policy-section": [ + { + "key": "enable_retention_policy_personal", + "type": "switch", + "label": "Personal workspaces", + "help": ( + "Each user chooses how long their own conversations and documents " + "are kept. Conversations idle and documents unchanged for longer " + "than that are deleted. Until a user chooses, the defaults below " + "apply." + ), + "default": False, + }, + { + "key": "default_retention_conversation_personal", + "type": "select", + "label": "Default conversation retention", + "help": "Applies to every user who has not chosen their own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_personal", "equals": True}, + }, + { + "key": "default_retention_document_personal", + "type": "select", + "label": "Default document retention", + "help": "Applies to every user who has not chosen their own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_personal", "equals": True}, + }, + { + "key": "enable_retention_policy_group", + "type": "switch", + "label": "Group workspaces", + "help": ( + "Group owners and admins choose how long the group's conversations " + "and documents are kept, including members' own chats grounded in " + "the group. Until they choose, the defaults below apply." + ), + "default": False, + }, + { + "key": "default_retention_conversation_group", + "type": "select", + "label": "Default conversation retention", + "help": "Applies to every group that has not chosen its own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_group", "equals": True}, + }, + { + "key": "default_retention_document_group", + "type": "select", + "label": "Default document retention", + "help": "Applies to every group that has not chosen its own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_group", "equals": True}, + }, + { + "key": "enable_retention_policy_public", + "type": "switch", + "label": "Public workspaces", + "help": ( + "Workspace owners and admins choose how long the workspace's " + "documents are kept, and the conversations grounded in it. Until " + "they choose, the defaults below apply." + ), + "default": False, + }, + { + "key": "default_retention_conversation_public", + "type": "select", + "label": "Default conversation retention", + "help": ( + "Applies to every public workspace that has not chosen its own " + "period. A workspace's conversation period covers every chat " + "grounded in it, including other users' chats with its documents, " + "in place of their owner's personal period." + ), + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_public", "equals": True}, + }, + { + "key": "default_retention_document_public", + "type": "select", + "label": "Default document retention", + "help": "Applies to every public workspace that has not chosen its own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_public", "equals": True}, + }, + { + # Shows the stored last and next run beside the hour, and writes the hour + # into the draft as an int. Validated on save by + # _COMPONENT_VALUE_VALIDATORS. + "key": "retention_policy_execution_hour", + "type": "component", + "component": "retention-schedule", + "label": "Daily run time", + "help": ( + "Retention runs once a day at this hour, in UTC. Saving a new hour, " + "or switching a workspace type on, reschedules the next run." + ), + "default": RETENTION_EXECUTION_HOUR_DEFAULT, + "depends_on": RETENTION_ANY_SCOPE_ENABLED, + }, + { + "type": "component", + "component": "retention-reset-defaults", + "label": "Reset to the defaults", + "help": ( + "Clears the period every user, group or public workspace of the " + "chosen types has set, so each follows the organization defaults " + "above. It cannot be undone; they would have to choose again." + ), + "depends_on": RETENTION_ANY_SCOPE_ENABLED, + }, + { + "type": "component", + "component": "retention-run-now", + "label": "Run retention now", + "help": ( + "Deletes everything already past its period straight away instead " + "of waiting for the daily run. Owners are told what was removed, " + "as they are after a scheduled run." + ), + "depends_on": RETENTION_ANY_SCOPE_ENABLED, + }, + ], + "document-classification-section": [ + { + "key": "enable_document_classification", + "type": "switch", + "role": "capability", + "label": "Enable document classification", + "help": ( + "Users label each document with one of the categories below. The " + "label shows as a coloured badge on the document and on the " + "conversations that cite it, and documents can be filtered by it." + ), + "default": False, + }, + { + # Writes document_classification_categories into the draft. Validated on + # save by _COMPONENT_VALUE_VALIDATORS. Required, so a section switched on + # with no categories reads as needing configuration: users would have + # nothing to choose from. + "key": "document_classification_categories", + "type": "component", + "component": "document-classification-categories", + "label": "Categories", + "help": ( + "Offered in this order wherever a user picks one. Renaming or " + "removing a category does not change documents already labelled " + "with it." + ), + "default": DOCUMENT_CLASSIFICATION_DEFAULT_CATEGORIES, + "required": True, + "depends_on": {"key": "enable_document_classification", "equals": True}, + }, + ], + "conversation-archiving-section": [ + { + "key": "enable_conversation_archiving", + "type": "switch", + "role": "capability", + "label": "Archive deleted conversations", + "help": ( + "A deleted conversation and its messages are copied to the archive " + "before they are removed, and a single deleted message is hidden " + "rather than erased. Archived copies are not shown to users; they " + "are kept for compliance and audit review." + ), + "default": False, + "notice": ( + "Retention follows this setting too: conversations past their period " + "are archived before they are removed. Documents are always deleted " + "permanently." + ), + "notice_level": "info", + }, + ], "cosmos-maintenance-section": [ { "key": "enable_app_maintenance", @@ -6193,6 +6456,9 @@ "inbound_mcp_allowed_client_app_entries": ["inbound_mcp_allowed_client_app_entries_json"], "inbound_mcp_allowed_tenant_entries": ["inbound_mcp_allowed_tenant_entries_json"], "inbound_mcp_allowed_source_entries": ["inbound_mcp_allowed_source_entries_json"], + # Same again for the classification categories, whose table script writes the + # list into a hidden JSON field; V2 edits the stored list directly. + "document_classification_categories": ["document_classification_categories_json"], } # Field names present in the V1 panes covered by a parity test that intentionally @@ -6960,6 +7226,139 @@ def merged(key): ) +def _validate_retention_execution_hour(value): + """Return ``(hour, error)`` for the daily retention run hour. + + The hour is stored as an int, because execute_retention_policy hands it to + ``datetime.replace(hour=...)``, which raises on a string. The classic form quietly + falls back to 2 AM for an out-of-range hour; refusing the value says so instead. + """ + message = ( + f"Choose an hour between {RETENTION_EXECUTION_HOUR_MIN} and " + f"{RETENTION_EXECUTION_HOUR_MAX}." + ) + if isinstance(value, bool): + return None, message + if isinstance(value, int): + hour = value + elif isinstance(value, float) and value.is_integer(): + hour = int(value) + elif isinstance(value, str) and re.fullmatch(r"\s*\d{1,2}\s*", value): + hour = int(value) + else: + return None, message + if not RETENTION_EXECUTION_HOUR_MIN <= hour <= RETENTION_EXECUTION_HOUR_MAX: + return None, message + return hour, None + + +def _normalize_document_classification_categories(value): + """Return ``(categories, error)`` for the document classification list. + + Applies the checks the classic save handler does -- a list of objects, each with + a non-empty label and a colour -- and tightens two of them. The colour must be a + six-digit hex, because it is drawn as the badge background. Two categories may + not share a label, because a document stores the label rather than an id, so + once applied the two could never be told apart. + """ + if not isinstance(value, list): + return None, "Expected a list of categories." + + categories = [] + first_use = {} + for index, item in enumerate(value, start=1): + if not isinstance(item, dict): + return None, f"Category {index} is not an object." + + raw_label = item.get("label") + label = raw_label.strip() if isinstance(raw_label, str) else "" + if not label: + return None, f"Category {index} needs a label." + if len(label) > DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH: + return None, ( + f"Category {index}'s label is longer than " + f"{DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH} characters." + ) + + raw_color = item.get("color") + color = raw_color.strip() if isinstance(raw_color, str) else "" + if not HEX_COLOR_PATTERN.match(color): + return None, f"Category {index} needs a hex colour such as #808080." + + folded = label.casefold() + if folded in first_use: + return None, ( + f'Categories {first_use[folded]} and {index} are both labelled ' + f'"{label}". Each label must be unique.' + ) + first_use[folded] = index + categories.append({"label": label, "color": color.lower()}) + + return categories, None + + +# Component-backed keys that are saved through the settings PATCH. A component +# normally owns its own persistence, so the type-driven path refuses a write to one; +# these are the exceptions. Each validator returns ``(value, error)``, which is what +# lets it refuse a value with a message -- the delegated normalizers below return a +# value only and have no way to say no. +_COMPONENT_VALUE_VALIDATORS = { + "retention_policy_execution_hour": _validate_retention_execution_hour, + "document_classification_categories": _normalize_document_classification_categories, +} + + +def compute_retention_next_run(hour, now=None): + """Return the next retention run at ``hour`` UTC, as an ISO timestamp. + + The rule the classic save handler and execute_retention_policy both apply: today + at that hour, or tomorrow once the hour has passed. + """ + current = now or datetime.now(timezone.utc) + next_run = current.replace(hour=hour, minute=0, second=0, microsecond=0) + if next_run <= current: + next_run += timedelta(days=1) + return next_run.isoformat() + + +def _apply_retention_schedule(normalized, current_settings, now=None): + """Reschedule the next retention run when a save changes what it depends on. + + The classic form recomputes ``retention_policy_next_run`` on every save. Without the + same step, a workspace type switched on here leaves no next run at all, and the + scheduler treats retention as due and runs it within minutes rather than at the + chosen hour. Only a save that changes a scope switch or the hour reschedules, so an + unrelated save never moves a pending run; a missing next run is filled in whenever + one of those keys is saved. + """ + submitted = [key for key in RETENTION_SCHEDULE_KEYS if key in normalized] + if not submitted: + return + + changed = any(normalized[key] != current_settings.get(key) for key in submitted) + if not changed and current_settings.get("retention_policy_next_run"): + return + + def merged(key, fallback): + if key in normalized: + return normalized[key] + stored = current_settings.get(key) + return fallback if stored is None else stored + + enabled = any( + _coerce_bool(merged(f"enable_retention_policy_{scope}", False)) + for scope in RETENTION_SCOPES + ) + if not enabled: + normalized["retention_policy_next_run"] = None + return + + hour, hour_error = _validate_retention_execution_hour( + merged("retention_policy_execution_hour", RETENTION_EXECUTION_HOUR_DEFAULT) + ) + if hour_error: + hour = RETENTION_EXECUTION_HOUR_DEFAULT + normalized["retention_policy_next_run"] = compute_retention_next_run(hour, now) # Keys whose normalization already exists elsewhere. Reusing those functions is @@ -7428,6 +7827,16 @@ def normalize_admin_settings_updates(updates, current_settings=None): normalized[key] = _DELEGATED_NORMALIZERS[key](value, field) continue + # Component-backed keys that are saved here, validated by a function that + # can refuse a value rather than coerce it. + if key in _COMPONENT_VALUE_VALIDATORS: + validated, validation_error = _COMPONENT_VALUE_VALIDATORS[key](value) + if validation_error: + errors[key] = validation_error + else: + normalized[key] = validated + continue + if field is None: normalized[key] = value continue @@ -7501,6 +7910,9 @@ def normalize_admin_settings_updates(updates, current_settings=None): if not errors: _apply_nested_paths(normalized, current) _apply_inbound_mcp_derivations(normalized, current) + # Derived from the merged scope switches and hour, so it is written only once + # the whole save is known to be valid. + _apply_retention_schedule(normalized, current) return normalized, errors, warnings diff --git a/application/single_app/config.py b/application/single_app/config.py index 670ab4808..3a890675e 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.259" +VERSION = "0.261.260" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_retention_policy.py b/application/single_app/functions_retention_policy.py index a9661ec97..612924359 100644 --- a/application/single_app/functions_retention_policy.py +++ b/application/single_app/functions_retention_policy.py @@ -6,13 +6,14 @@ This module handles automated deletion of aged conversations and documents based on configurable retention policies for personal, group, and public workspaces. -Version: 0.261.038 +Version: 0.261.260 Implemented in: 0.234.067 Updated in: 0.236.012 - Fixed race condition handling for NotFound errors during deletion Updated in: 0.237.004 - Fixed critical bug where conversations with null/undefined last_activity_at were deleted regardless of age Updated in: 0.237.005 - Fixed field name: use last_updated (actual field) instead of last_activity_at (non-existent) Updated in: 0.250.103 - Applied retention by conversation ownership across current, legacy, and collaboration stores Updated in: 0.261.038 - Stop pending Microsoft 365 delivery before deleting its destination +Updated in: 0.261.260 - Chats grounded in a public workspace follow that workspace's conversation policy while it exists """ from config import * @@ -30,6 +31,7 @@ from functions_debug import debug_print from functions_appinsights import log_event from datetime import datetime, timezone, timedelta +import logging GROUP_SINGLE_USER_CHAT_TYPES = {'group', 'group-single-user', 'group_single_user'} @@ -97,6 +99,33 @@ def get_all_public_workspaces(): return [] +def _get_existing_public_workspace_ids(): + """Return the ids of the public workspaces that exist, or None if they cannot be listed. + + None is deliberately distinct from an empty set. A failed or partial listing must never + read as every public workspace having been deleted, which would hand every grounded chat + to its owner's personal policy. + """ + try: + workspace_ids = set() + for row in cosmos_public_workspaces_container.query_items( + query="SELECT c.id FROM c", + enable_cross_partition_query=True, + ): + workspace_id = str((row or {}).get('id') or '').strip() + if workspace_id: + workspace_ids.add(workspace_id) + return frozenset(workspace_ids) + except Exception as e: + log_event( + "[RETENTION_POLICY] Public workspace listing failed", + {"error_type": type(e).__name__}, + level=logging.ERROR, + ) + debug_print(f"[RETENTION_POLICY] Could not list public workspaces: {e}") + return None + + def resolve_retention_value(value, workspace_type, retention_type, settings=None): """ Resolve a retention value, handling 'default' by looking up organization defaults. @@ -173,6 +202,36 @@ def _is_group_single_user_conversation(conversation_item): return chat_type in GROUP_SINGLE_USER_CHAT_TYPES or bool(_get_primary_group_id(conversation_item)) +def _get_primary_public_workspace_id(conversation_item): + """Return the public workspace a single-user conversation is grounded in.""" + for context_item in list((conversation_item or {}).get('context', []) or []): + if not isinstance(context_item, dict): + continue + if context_item.get('type') != 'primary' or context_item.get('scope') != 'public': + continue + return str(context_item.get('id') or '').strip() + return '' + + +def _is_governed_by_public_workspace(conversation_item, existing_public_workspace_ids): + """Whether a public workspace's policy, rather than the owner's, governs a conversation. + + Public workspaces have no conversation store; a chat grounded in one is a personal + conversation whose primary context names the workspace. It follows that workspace's + policy while the workspace exists. A chat with no workspace id, or whose workspace has + been deleted, stays under its owner's personal policy instead of falling outside every + policy. When the existing workspaces are unknown (``None``), every grounded chat is + treated as governed, so a personal run skips it rather than risk deleting it under the + wrong policy. + """ + public_workspace_id = _get_primary_public_workspace_id(conversation_item) + if not public_workspace_id: + return False + if existing_public_workspace_ids is None: + return True + return public_workspace_id in existing_public_workspace_ids + + def _is_converted_conversation_source(conversation_item): return bool(str((conversation_item or {}).get('collaboration_conversation_id') or '').strip()) @@ -204,14 +263,36 @@ def _build_group_scope_query(timestamp_field): """ +def _build_public_scope_query(timestamp_field): + return f""" + SELECT * FROM c + WHERE IS_ARRAY(c.context) + AND EXISTS( + SELECT VALUE context_item + FROM context_item IN c.context + WHERE context_item.type = 'primary' + AND context_item.scope = 'public' + AND context_item.id = @scope_id + ) + AND IS_DEFINED(c.{timestamp_field}) + AND IS_STRING(c.{timestamp_field}) + AND c.{timestamp_field} < @cutoff_date + """ + + def _build_conversation_retention_sources( workspace_type, cutoff_iso, user_id=None, group_id=None, public_workspace_id=None, + existing_public_workspace_ids=None, ): - """Describe the backing stores governed by a workspace retention policy.""" + """Describe the backing stores governed by a workspace retention policy. + + ``existing_public_workspace_ids`` lets a personal run take back chats grounded in a + public workspace that has since been deleted. Without it, grounded chats are skipped. + """ if workspace_type == 'personal': return [ { @@ -232,6 +313,10 @@ def _build_conversation_retention_sources( ], 'matches_scope': lambda item: ( not _is_group_single_user_conversation(item) + and not _is_governed_by_public_workspace( + item, + existing_public_workspace_ids, + ) and not _is_converted_conversation_source(item) ), }, @@ -316,24 +401,25 @@ def _build_conversation_retention_sources( }, ] + # Public workspaces have no conversation store of their own. A chat grounded in + # one is a personal conversation whose primary context names the workspace, and + # it follows that workspace's policy the way a group-grounded chat follows its + # group's. return [ { - 'name': 'public', - 'container': cosmos_public_conversations_container, - 'messages_container': cosmos_public_messages_container, + 'name': 'public_single_user', + 'container': cosmos_conversations_container, + 'messages_container': cosmos_messages_container, 'timestamp_field': 'last_updated', - 'query': """ - SELECT * FROM c - WHERE c.public_workspace_id = @scope_id - AND IS_DEFINED(c.last_updated) - AND IS_STRING(c.last_updated) - AND c.last_updated < @cutoff_date - """, + 'query': _build_public_scope_query('last_updated'), 'parameters': [ {'name': '@scope_id', 'value': public_workspace_id}, {'name': '@cutoff_date', 'value': cutoff_iso}, ], - 'matches_scope': lambda item: item.get('public_workspace_id') == public_workspace_id, + 'matches_scope': lambda item: ( + _get_primary_public_workspace_id(item) == public_workspace_id + and not _is_converted_conversation_source(item) + ), }, ] @@ -361,6 +447,7 @@ def _delete_standard_conversation_for_retention( return { 'id': conversation_id, 'title': conversation_title, + 'user_id': selected_conversation_item.get('user_id'), source['timestamp_field']: selected_conversation_item.get( source['timestamp_field'] ), @@ -385,6 +472,12 @@ def _delete_standard_conversation_for_retention( conversation_item = live_conversation_item cancel_m365_conversation_deliveries(conversation_id) + # A grounded chat names its workspace in its primary context, not at the top level. + grounded_public_workspace_id = ( + _get_primary_public_workspace_id(conversation_item) + or conversation_item.get('public_workspace_id') + or None + ) if archiving_enabled: archived_item = dict(conversation_item) @@ -401,7 +494,7 @@ def _delete_standard_conversation_for_retention( context=conversation_item.get('context', []), tags=conversation_item.get('tags', []), group_id=_get_primary_group_id(conversation_item) or None, - public_workspace_id=conversation_item.get('public_workspace_id'), + public_workspace_id=grounded_public_workspace_id, additional_context={'deletion_reason': 'retention_policy'}, ) @@ -456,7 +549,7 @@ def _delete_standard_conversation_for_retention( is_archived=archiving_enabled, is_bulk_operation=True, group_id=_get_primary_group_id(conversation_item) or None, - public_workspace_id=conversation_item.get('public_workspace_id'), + public_workspace_id=grounded_public_workspace_id, additional_context={ 'deletion_reason': 'retention_policy', 'retention_source': source['name'], @@ -480,6 +573,8 @@ def _delete_standard_conversation_for_retention( return { 'id': conversation_id, 'title': conversation_title, + # Lets a public workspace's run tell each owner about their own chats. + 'user_id': conversation_item.get('user_id'), source['timestamp_field']: conversation_item.get(source['timestamp_field']), } @@ -594,6 +689,9 @@ def process_personal_retention(): # Pre-load settings once for efficiency settings = get_settings() + + # Chats grounded in a public workspace follow its policy only while it exists. + existing_public_workspace_ids = _get_existing_public_workspace_ids() for user in all_users: user_id = user.get('id') @@ -632,7 +730,8 @@ def process_personal_retention(): conv_results = delete_aged_conversations( user_id=user_id, retention_days=int(conversation_retention_days), - workspace_type='personal' + workspace_type='personal', + existing_public_workspace_ids=existing_public_workspace_ids, ) user_deletion_summary['conversations_deleted'] = conv_results['count'] user_deletion_summary['conversation_details'] = conv_results['details'] @@ -814,11 +913,26 @@ def process_public_retention(): 'document_details': [] } - # Note: Public workspaces do not have a separate conversations container. - # Conversations are only stored in personal (cosmos_conversations_container) or - # group (cosmos_group_conversations_container) workspaces. - # Therefore, we skip conversation processing for public workspaces. - # Only documents are processed for public workspace retention. + # Public workspaces have no conversation store of their own. The chats + # this processes are personal conversations grounded in the workspace, + # which follow its policy the way group-grounded chats follow their group's. + if conversation_retention_days != 'none': + try: + conv_results = delete_aged_conversations( + public_workspace_id=workspace_id, + retention_days=int(conversation_retention_days), + workspace_type='public' + ) + workspace_deletion_summary['conversations_deleted'] = conv_results['count'] + workspace_deletion_summary['conversation_details'] = conv_results['details'] + results['conversations'] += conv_results['count'] + except Exception as e: + log_event( + "[RETENTION_POLICY] Public workspace conversation retention failed", + {"error_type": type(e).__name__, "public_workspace_id": workspace_id}, + level=logging.ERROR, + ) + debug_print(f"Error processing conversations for public workspace {workspace_id}: {e}") # Process documents if document_retention_days != 'none': @@ -837,7 +951,18 @@ def process_public_retention(): # Send notification if anything was deleted if workspace_deletion_summary['conversations_deleted'] > 0 or workspace_deletion_summary['documents_deleted'] > 0: - send_retention_notification(workspace_id, workspace_deletion_summary, 'public') + # The chats belong to individual users, so the workspace's managers + # are told how many were removed and each owner is told which. + send_retention_notification( + workspace_id, + {**workspace_deletion_summary, 'conversation_details': []}, + 'public', + ) + send_public_conversation_owner_notifications( + workspace_id, + workspace_deletion_summary['workspace_name'], + workspace_deletion_summary['conversation_details'], + ) results['workspaces_affected'] += 1 results['details'].append(workspace_deletion_summary) @@ -849,7 +974,14 @@ def process_public_retention(): return results -def delete_aged_conversations(retention_days, workspace_type='personal', user_id=None, group_id=None, public_workspace_id=None): +def delete_aged_conversations( + retention_days, + workspace_type='personal', + user_id=None, + group_id=None, + public_workspace_id=None, + existing_public_workspace_ids=None, +): """ Delete conversations governed by a workspace policy across all backing stores. @@ -859,6 +991,9 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id user_id (str, optional): User ID for personal workspaces group_id (str, optional): Group ID for group workspaces public_workspace_id (str, optional): Public workspace ID for public workspaces + existing_public_workspace_ids (frozenset, optional): For personal runs, the public + workspaces that exist. Chats grounded in any other workspace return to their + owner's policy; when omitted, every grounded chat is skipped. Returns: dict: {'count': int, 'details': list} @@ -874,6 +1009,7 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id user_id=user_id, group_id=group_id, public_workspace_id=public_workspace_id, + existing_public_workspace_ids=existing_public_workspace_ids, ) for source in sources: @@ -949,6 +1085,7 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id deleted_details.append({ 'id': conversation_id, 'title': conversation_item.get('title', 'Untitled'), + 'user_id': conversation_item.get('user_id'), source['timestamp_field']: conversation_item.get(source['timestamp_field']), 'already_deleted': True, }) @@ -1183,3 +1320,60 @@ def send_retention_notification(workspace_id, deletion_summary, workspace_type): ) debug_print(f"Sent retention notification to {workspace_type} workspace {workspace_id}") + + +def send_public_conversation_owner_notifications(public_workspace_id, workspace_name, conversation_details): + """Tell each owner which of their chats a public workspace's policy removed. + + A chat grounded in a public workspace is a personal conversation, so its title + belongs to its owner. The workspace's own notification carries only a count; the + titles go here, to each owner, and nowhere else. Records another process had + already deleted are left out, because retention did not remove them. + """ + deleted_by_owner = {} + for detail in conversation_details or []: + if not isinstance(detail, dict) or detail.get('already_deleted'): + continue + owner_id = str(detail.get('user_id') or '').strip() + if owner_id: + deleted_by_owner.setdefault(owner_id, []).append(detail) + + display_name = str(workspace_name or '').strip() or 'a public workspace' + for owner_id, owner_details in deleted_by_owner.items(): + count = len(owner_details) + lines = [ + f"Retention policy for {display_name} automatically deleted {count} " + f"of your conversation{'s' if count != 1 else ''} grounded in it.", + "", + "**Conversations:**", + ] + for detail in owner_details[:10]: + lines.append(f"• {detail.get('title', 'Untitled')}") + if count > 10: + lines.append(f"• ...and {count - 10} more") + + try: + create_notification( + user_id=owner_id, + notification_type='system_announcement', + title='Retention Policy Cleanup', + message="\n".join(lines), + link_url='/chats', + metadata={ + 'conversations_deleted': count, + 'documents_deleted': 0, + 'public_workspace_id': public_workspace_id, + 'deletion_date': datetime.now(timezone.utc).isoformat(), + }, + ) + except Exception as notification_error: + # One owner's notification failing must not stop the others. + log_event( + "[RETENTION_POLICY] Owner notification for a public workspace run failed", + { + "error_type": type(notification_error).__name__, + "public_workspace_id": public_workspace_id, + "user_id": owner_id, + }, + level=logging.WARNING, + ) diff --git a/application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx b/application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx new file mode 100644 index 000000000..a602377c1 --- /dev/null +++ b/application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx @@ -0,0 +1,210 @@ +// ClassificationCategoriesEditor.tsx +// The categories users choose from when they classify a document. +// +// The classic pane edits these in a table with an Edit and a Save button per row, and +// keeps the list in a hidden JSON field. Here each row is edited in place and the list +// flows into the page's draft like any other setting, so the save bar saves it with the +// rest of the section. +// +// Order matters: it is the order users see wherever they pick a category, so rows move. +// The preview draws each category with the same badge documents and conversations use. + +import { clsx } from 'clsx'; +import { AlertCircle, ArrowDown, ArrowUp, Plus, Tags, Trash2 } from 'lucide-react'; +import type { AdminField } from '../../lib/adminFields'; +import { + CLASSIFICATION_DEFAULT_COLOR, + CLASSIFICATION_LABEL_MAX_LENGTH, + describeCategoryProblems, + readClassificationCategories, + safeBadgeColor, + type ClassificationCategory, +} from '../../lib/classificationCategories'; +import { ClassificationBadge } from '../documents/documentPresentation'; +import { GlassButton } from '../ui/primitives'; + +const inputClass = clsx( + 'min-h-9 rounded-lg border border-edge bg-surface-1 px-2.5 py-1.5', + 'text-sm text-text-1 placeholder:text-text-3', + 'focus:border-accent focus:outline-none', + 'disabled:cursor-not-allowed disabled:opacity-60', +); + +const iconButtonClass = clsx( + 'rounded-lg p-1.5 text-text-3 transition-colors hover:bg-surface-2 hover:text-text-1', + 'disabled:cursor-not-allowed disabled:opacity-40', +); + +export function ClassificationCategoriesEditor({ + field, + value, + error, + disabled, + onChange, +}: { + field: AdminField; + value: unknown; + error?: string; + disabled?: boolean; + onChange: (next: ClassificationCategory[]) => void; +}) { + const categories = readClassificationCategories(value); + const problems = describeCategoryProblems(categories); + const previewable = categories.filter( + (category, index) => !problems[index] && category.label.trim(), + ); + + const replace = (index: number, patch: Partial) => { + onChange(categories.map((category, i) => (i === index ? { ...category, ...patch } : category))); + }; + + const move = (index: number, delta: number) => { + const target = index + delta; + if (target < 0 || target >= categories.length) { + return; + } + const next = [...categories]; + [next[index], next[target]] = [next[target], next[index]]; + onChange(next); + }; + + return ( +
+
+ {field.label} + + {categories.length} {categories.length === 1 ? 'category' : 'categories'} + +
+ + {field.help ? ( +

{field.help}

+ ) : null} + + {categories.length === 0 ? ( +
+
+ ) : ( +
    + {categories.map((category, index) => { + const problem = problems[index]; + const swatch = safeBadgeColor(category.color) ?? CLASSIFICATION_DEFAULT_COLOR; + const position = index + 1; + return ( +
  • + {/* The label leads, so DOM order matches what is seen: one line on a + wide card; on a narrow one, or at a large text size, the label + takes its own line and the rest wrap beneath it. */} +
    + replace(index, { label: event.target.value })} + /> + replace(index, { color: event.target.value })} + /> + replace(index, { color: event.target.value })} + /> +
    + + + +
    +
    + + {problem ? ( +

    +

    + ) : null} +
    • + ); + })} +
+ )} + + onChange([...categories, { label: '', color: CLASSIFICATION_DEFAULT_COLOR }])} + > + + + {previewable.length ? ( +
+

As users will see them

+
+ {previewable.map((category, index) => ( + + ))} +
+
+ ) : null} + + {error ? ( +

+

+ ) : null} +
+ ); +} diff --git a/application/v2_ui/src/components/admin/RetentionOperations.tsx b/application/v2_ui/src/components/admin/RetentionOperations.tsx new file mode 100644 index 000000000..74a19eef4 --- /dev/null +++ b/application/v2_ui/src/components/admin/RetentionOperations.tsx @@ -0,0 +1,709 @@ +// RetentionOperations.tsx +// Run retention now, and reset every chosen period back to the organization defaults. +// +// Both act on everything stored for a workspace type and neither can be undone, so a click +// opens a review rather than acting: the types it may touch, the defaults those types +// follow, and plainly what happens to the data. The review sits inline rather than in a +// dialog so the defaults above stay in view, and so a run that takes minutes on a large +// deployment does not hold the whole page behind a modal. +// +// Both work from saved settings only, because the routes read what is stored. While the +// draft holds an edit they depend on, both are held with a message saying to save first, +// and only workspace types whose retention is on in the saved settings can be chosen. + +import { useEffect, useId, useRef, useState, type ReactNode, type RefObject } from 'react'; +import { clsx } from 'clsx'; +import { + AlertCircle, + AlertTriangle, + CheckCircle2, + CirclePlay, + Loader2, + RotateCcw, + XCircle, + type LucideIcon, +} from 'lucide-react'; +import { api } from '../../lib/apiClient'; +import { asBoolean, type AdminField } from '../../lib/adminFields'; +import { + CONVERSATION_ARCHIVING_KEY, + RETENTION_AFFECTED_NOUNS, + RETENTION_LAST_RUN_KEY, + RETENTION_NEXT_RUN_KEY, + RETENTION_RUN_KEYS, + RETENTION_SCOPES, + RETENTION_SCOPE_LABELS, + RETENTION_SETTING_KEYS, + retentionRequestError, + savedEnabledScopes, + scopeDefaultsSummary, + summarizeRetentionReset, + summarizeRetentionRun, + unsavedKeys, + type RetentionResetSummary, + type RetentionRunSummary, + type RetentionScope, +} from '../../lib/retentionPolicy'; +import type { Json } from '../../lib/types'; +import { GlassButton } from '../ui/primitives'; +import { RETENTION_SCOPE_ICONS } from './retentionScopeIcons'; + +type Phase = 'idle' | 'review' | 'working' | 'done' | 'failed'; + +function plural(count: number, [one, many]: readonly [string, string]): string { + return `${count.toLocaleString()} ${count === 1 ? one : many}`; +} + +/** A row laid out like every other setting: label and help on the left, action on the right. */ +function OperationRow({ field, children }: { field: AdminField; children: ReactNode }) { + return ( +
+
{field.label}
+ {field.help ? ( +

{field.help}

+ ) : null} +
{children}
+
+ ); +} + +function Callout({ tone, children }: { tone: 'warn' | 'danger' | 'ok'; children: ReactNode }) { + const Icon: LucideIcon = tone === 'ok' ? CheckCircle2 : tone === 'danger' ? XCircle : AlertTriangle; + return ( +
+
+ ); +} + +/** Why an action is held, shown under its trigger and inside an open review. */ +function HeldNotice({ children }: { children: ReactNode }) { + return ( +

+

+ ); +} + +function ScopeChoices({ + idPrefix, + legend, + enabled, + selected, + disabled, + describe, + onToggle, +}: { + idPrefix: string; + legend: string; + enabled: readonly RetentionScope[]; + selected: readonly RetentionScope[]; + disabled: boolean; + describe: (scope: RetentionScope) => ReactNode; + onToggle: (scope: RetentionScope, checked: boolean) => void; +}) { + return ( +
+ {legend} +
    + {RETENTION_SCOPES.map((scope) => { + const available = enabled.includes(scope); + const Icon = RETENTION_SCOPE_ICONS[scope]; + const id = `${idPrefix}-${scope}`; + return ( +
  • + onToggle(scope, event.target.checked)} + /> + +
    • + ); + })} +
+
+ ); +} + +/** + * The open/close, focus and selection behaviour both reviews share. + * + * Opening moves focus to the review's heading so a keyboard or screen reader user lands in + * it; closing hands focus back to the button that opened it. + */ +function useReview(enabled: readonly RetentionScope[]) { + const [phase, setPhase] = useState('idle'); + const [selected, setSelected] = useState([]); + const triggerRef = useRef(null); + const headingRef = useRef(null); + const returnFocus = useRef(false); + + useEffect(() => { + if (phase === 'review' || phase === 'done' || phase === 'failed') { + headingRef.current?.focus(); + } else if (phase === 'idle' && returnFocus.current) { + returnFocus.current = false; + triggerRef.current?.focus(); + } + }, [phase]); + + return { + phase, + setPhase, + // A type switched off and saved while the review is open drops out of the choice. + chosen: selected.filter((scope) => enabled.includes(scope)), + triggerRef, + headingRef, + open: () => { + setSelected([]); + setPhase('review'); + }, + close: () => { + returnFocus.current = true; + setPhase('idle'); + }, + toggle: (scope: RetentionScope, checked: boolean) => + setSelected((current) => + checked ? [...new Set([...current, scope])] : current.filter((item) => item !== scope), + ), + }; +} + +/** + * The button that opens a review, styled like the other in-row actions (Test connection). + * + * It grows with its label rather than holding a fixed height, so at a large text size on a + * narrow card the label wraps inside the button instead of spilling over the help text. + */ +function ReviewTrigger({ + triggerRef, + icon: Icon, + label, + disabled, + onOpen, +}: { + triggerRef: RefObject; + icon: LucideIcon; + label: string; + disabled: boolean; + onOpen: () => void; +}) { + return ( + + ); +} + +interface OperationProps { + field: AdminField; + /** The saved settings. Both actions work from these alone. */ + settings: Json; + draft: Json; + /** True while the page is saving. */ + disabled?: boolean; +} + +export function RetentionRunNow({ + field, + settings, + draft, + disabled = false, + onStoredSettingsChange, + onOpenSection, +}: OperationProps & { + /** Merge the run's new last and next run into the page's copy of the saved settings. */ + onStoredSettingsChange: (partial: Json) => void; + onOpenSection: (sectionId: string) => void; +}) { + const enabled = savedEnabledScopes(settings); + const held = unsavedKeys(draft, RETENTION_RUN_KEYS).length > 0; + const archiving = asBoolean(settings[CONVERSATION_ARCHIVING_KEY]); + const review = useReview(enabled); + const [summary, setSummary] = useState(null); + const [failure, setFailure] = useState(''); + const idPrefix = useId(); + const headingId = `${idPrefix}-heading`; + + const refreshSchedule = async () => { + try { + const response = await api.get<{ settings?: Json }>('/api/admin/retention-policy/settings'); + const stored = response.settings ?? {}; + onStoredSettingsChange({ + [RETENTION_LAST_RUN_KEY]: stored[RETENTION_LAST_RUN_KEY] ?? null, + [RETENTION_NEXT_RUN_KEY]: stored[RETENTION_NEXT_RUN_KEY] ?? null, + }); + } catch { + // The run's own outcome is already on screen; a stale schedule readout is not + // worth a second error beside it, and "Check again" can read it later. + } + }; + + const run = async () => { + const scopes = review.chosen; + if (!scopes.length || held) { + return; + } + review.setPhase('working'); + setSummary(null); + setFailure(''); + try { + const response = await api.post('/api/admin/retention-policy/execute', { scopes }); + setSummary(summarizeRetentionRun(response, scopes)); + review.setPhase('done'); + } catch (error) { + setFailure(retentionRequestError(error, 'run')); + review.setPhase('failed'); + } finally { + void refreshSchedule(); + } + }; + + if (review.phase === 'idle') { + return ( + + + {held ? ( + + Save or discard your changes first. Retention runs on saved settings only. + + ) : null} + + ); + } + + const working = review.phase === 'working'; + + return ( + +
+ {review.phase === 'done' && summary ? ( + <> +

+ {summary.ok && !summary.errorCount ? 'Retention finished' : 'Retention finished with problems'} +

+
+ +

+ {summary.conversations || summary.documents + ? `Removed ${plural(summary.conversations, ['conversation', 'conversations'])} and ${plural(summary.documents, ['document', 'documents'])}.` + : 'Nothing was past its period, so nothing was removed.'} +

+ {summary.errorCount ? ( +

+ The run reported {plural(summary.errorCount, ['problem', 'problems'])}. Check + the application logs for the cause before running it again. +

+ ) : null} + + {summary.rows.length ? ( +
+ + + + + + + + + + + {summary.rows.map((row) => ( + + + + + + + ))} + +
Workspace typeConversationsDocumentsAffected
+ {RETENTION_SCOPE_LABELS[row.scope]} + {row.conversations.toLocaleString()}{row.documents.toLocaleString()} + {plural(row.affected, RETENTION_AFFECTED_NOUNS[row.scope])} +
+
+ ) : null} +
+
+ + Done + +
+ + ) : review.phase === 'failed' ? ( + <> +

+ Retention did not report back +

+
+ +

{failure}

+ +
+
+ + Close + +
+ + ) : ( + <> +

+ Review before running +

+

+ Every user, group or workspace keeps the period it chose. The rest follow + the defaults shown. +

+ scopeDefaultsSummary(settings, scope, 'run')} + onToggle={review.toggle} + /> +
+ +

+ + This deletes data now and cannot be undone. + {' '} + Conversations idle and documents unchanged for longer than their + period are removed from the chosen types. +

+

Documents are deleted permanently, with their search entries and stored files.

+

+ {archiving + ? 'Conversations are copied to the archive first, because Conversation Archiving is on. ' + : 'Conversations are deleted permanently, because Conversation Archiving is off. '} + +

+

Affected owners are notified of what was removed.

+ +
+ {held ? ( + + Save or discard your changes first. Retention runs on saved settings only. + + ) : null} + {working ? ( +

+ Running. On a large deployment this can take several minutes; the result + appears here when it finishes. +

+ ) : null} +
+ + Cancel + + void run()} + > + {working ? ( + +
+ + )} +
+ + ); +} + +export function RetentionResetDefaults({ field, settings, draft, disabled = false }: OperationProps) { + const enabled = savedEnabledScopes(settings); + const held = unsavedKeys(draft, RETENTION_SETTING_KEYS).length > 0; + const review = useReview(enabled); + const [summary, setSummary] = useState(null); + const [failure, setFailure] = useState(''); + const idPrefix = useId(); + const headingId = `${idPrefix}-heading`; + + const reset = async () => { + const scopes = review.chosen; + if (!scopes.length || held) { + return; + } + review.setPhase('working'); + setSummary(null); + setFailure(''); + try { + const response = await api.post('/api/admin/retention-policy/force-push', { scopes }); + setSummary(summarizeRetentionReset(response, scopes)); + review.setPhase('done'); + } catch (error) { + setFailure(retentionRequestError(error, 'reset')); + review.setPhase('failed'); + } + }; + + if (review.phase === 'idle') { + return ( + + + {held ? ( + + Save or discard your changes first. The reset applies the saved defaults only. + + ) : null} + + ); + } + + const working = review.phase === 'working'; + + return ( + +
+ {review.phase === 'done' && summary ? ( + <> +

+ {summary.ok ? 'Defaults applied' : 'The reset did not finish'} +

+
+ +

+ {summary.ok + ? `Reset ${plural(summary.total, ['retention policy', 'retention policies'])} to follow the organization defaults.` + : 'Some policies may already follow the defaults. Check the application logs, then try again.'} +

+ + {summary.rows.length ? ( +
    + {summary.rows.map((row) => ( +
  • + {RETENTION_SCOPE_LABELS[row.scope]} + + {plural(row.updated, RETENTION_AFFECTED_NOUNS[row.scope])} + +
    • + ))} +
+ ) : null} +
+
+ + Done + +
+ + ) : review.phase === 'failed' ? ( + <> +

+ The reset did not report back +

+
+ +

{failure}

+ +
+
+ + Close + +
+ + ) : ( + <> +

+ Review before resetting +

+

+ Everyone in the chosen types goes back to following these defaults. +

+ scopeDefaultsSummary(settings, scope, 'reset')} + onToggle={review.toggle} + /> +
+ +

+ This cannot be undone.{' '} + Every user, group and public workspace of the chosen types loses the + period it chose. They would have to choose again. +

+

Nothing is deleted by the reset itself; the next run applies the defaults.

+ +
+ {held ? ( + + Save or discard your changes first. The reset applies the saved defaults only. + + ) : null} + {working ? ( +

+ Resetting. This visits every user, group or workspace of the chosen types. +

+ ) : null} +
+ + Cancel + + void reset()} + > + {working ? ( + +
+ + )} +
+ + ); +} diff --git a/application/v2_ui/src/components/admin/RetentionSchedule.tsx b/application/v2_ui/src/components/admin/RetentionSchedule.tsx new file mode 100644 index 000000000..1e8b512e9 --- /dev/null +++ b/application/v2_ui/src/components/admin/RetentionSchedule.tsx @@ -0,0 +1,191 @@ +// RetentionSchedule.tsx +// The daily retention run: when it happens, when it last happened, and when it next will. +// +// The classic pane shows the hour as a select and the last and next run as raw ISO +// strings. Here the hour reads as UTC with the administrator's own clock beside it, and +// both timestamps read as a date plus how far away they are, because "next run in 3 +// hours" is what someone checking the schedule actually wants to know. +// +// The hour is stored as an int -- execute_retention_policy hands it to +// datetime.replace(hour=...) -- which is why this is a component rather than a generic +// select, whose options are strings. + +import { useState, type ReactNode } from 'react'; +import { clsx } from 'clsx'; +import { AlertCircle, Loader2, RefreshCw } from 'lucide-react'; +import { api } from '../../lib/apiClient'; +import type { AdminField } from '../../lib/adminFields'; +import { + RETENTION_LAST_RUN_KEY, + RETENTION_NEXT_RUN_KEY, + describeRunTime, + localTimeForUtcHour, + projectNextRun, + readExecutionHour, + utcHourLabel, + willReschedule, +} from '../../lib/retentionPolicy'; +import type { Json } from '../../lib/types'; +import { FieldShell, inputClass } from './fields'; + +const HOURS = Array.from({ length: 24 }, (_, hour) => hour); + +interface ScheduleResponse { + settings?: Json; +} + +export function RetentionSchedule({ + field, + value, + settings, + draft, + error, + disabled, + onChange, + onStoredSettingsChange, +}: { + field: AdminField; + value: unknown; + /** The saved settings, which hold the last and next run. */ + settings: Json; + draft: Json; + error?: string; + disabled?: boolean; + onChange: (hour: number) => void; + /** Merge freshly read schedule values into the page's copy of the saved settings. */ + onStoredSettingsChange: (partial: Json) => void; +}) { + const [checking, setChecking] = useState(false); + const [checkFailed, setCheckFailed] = useState(false); + + const id = `admin-field-${field.key ?? 'retention-schedule'}`; + const hour = readExecutionHour(value); + const now = new Date(); + const local = localTimeForUtcHour(hour, now); + const last = describeRunTime(settings[RETENTION_LAST_RUN_KEY], now); + const next = describeRunTime(settings[RETENTION_NEXT_RUN_KEY], now); + const projected = willReschedule(settings, draft) + ? describeRunTime(projectNextRun(hour, now), now) + : null; + + const check = async () => { + setChecking(true); + setCheckFailed(false); + try { + const response = await api.get('/api/admin/retention-policy/settings'); + const stored = response.settings ?? {}; + onStoredSettingsChange({ + [RETENTION_LAST_RUN_KEY]: stored[RETENTION_LAST_RUN_KEY] ?? null, + [RETENTION_NEXT_RUN_KEY]: stored[RETENTION_NEXT_RUN_KEY] ?? null, + }); + } catch { + setCheckFailed(true); + } finally { + setChecking(false); + } + }; + + let nextRun: ReactNode; + if (projected) { + nextRun = ( + <> + + · {projected.relative}, once you save + + ); + } else if (next && next.past) { + nextRun = ( + <> + + · due now; the scheduler checks every 5 minutes + + ); + } else if (next) { + nextRun = ( + <> + + · {next.relative} + + ); + } else { + nextRun = ( + + Not set yet. It is worked out when a run time is saved or the next run finishes. + + ); + } + + return ( + +
+ + {local ? ( +

That is {local} on your clock.

+ ) : null} +
+ +
+ {/* Each pair wraps its value below the term once the card is too narrow to + hold both on one line, rather than squeezing the value into a sliver. */} +
+
Last run
+
+ {last ? ( + <> + + · {last.relative} + + ) : ( + Never run + )} +
+
+
+
Next run
+
{nextRun}
+
+
+ +
+ + {checkFailed ? ( + + + ) : null} +
+ + ); +} diff --git a/application/v2_ui/src/components/admin/agentSectionAppearance.ts b/application/v2_ui/src/components/admin/agentSectionAppearance.ts index 3e19d5e6e..fd3b79df6 100644 --- a/application/v2_ui/src/components/admin/agentSectionAppearance.ts +++ b/application/v2_ui/src/components/admin/agentSectionAppearance.ts @@ -2,13 +2,15 @@ import { UserRound, UsersRound } from 'lucide-react'; import type { SettingsSectionAppearance } from './SettingsSection'; +import { RETENTION_SCOPE_ICONS } from './retentionScopeIcons'; // Presentation only: field order, visibility, and status still come from the schema. // // Section icons now come from the navigation definition, and which switch leads which // settings is derived from `depends_on`, so the Agent Runtime emphasis no longer needs -// declaring here. What remains is the one cue the schema cannot express: whether a -// workspace permission applies to a person or to a group. +// declaring here. What remains are cues the schema cannot express: whether a workspace +// permission applies to a person or to a group, and which workspace type each retention +// switch governs. export const agentSectionAppearances: Readonly< Partial> > = { @@ -21,4 +23,13 @@ export const agentSectionAppearances: Readonly< allow_group_custom_endpoints: { Icon: UsersRound }, }, }, + // The three workspace types are peers, so the first is not promoted to the switch the + // card is about, which is what the derived hierarchy would otherwise do with it. + 'retention-policy-section': { + fields: { + enable_retention_policy_personal: { Icon: RETENTION_SCOPE_ICONS.personal, emphasis: 'none' }, + enable_retention_policy_group: { Icon: RETENTION_SCOPE_ICONS.group }, + enable_retention_policy_public: { Icon: RETENTION_SCOPE_ICONS.public }, + }, + }, }; diff --git a/application/v2_ui/src/components/admin/retentionScopeIcons.ts b/application/v2_ui/src/components/admin/retentionScopeIcons.ts new file mode 100644 index 000000000..9560a0bf8 --- /dev/null +++ b/application/v2_ui/src/components/admin/retentionScopeIcons.ts @@ -0,0 +1,12 @@ +// retentionScopeIcons.ts +// One icon per workspace type, shared by the Retention Policy switches and the run and +// reset reviews so a type looks the same wherever it is named. + +import { Globe, User, Users, type LucideIcon } from 'lucide-react'; +import type { RetentionScope } from '../../lib/retentionPolicy'; + +export const RETENTION_SCOPE_ICONS: Readonly> = { + personal: User, + group: Users, + public: Globe, +}; diff --git a/application/v2_ui/src/lib/classificationCategories.ts b/application/v2_ui/src/lib/classificationCategories.ts new file mode 100644 index 000000000..e3109cc6e --- /dev/null +++ b/application/v2_ui/src/lib/classificationCategories.ts @@ -0,0 +1,75 @@ +// classificationCategories.ts +// Reading and checking the document classification categories edited in Admin Settings. +// +// The server is authoritative (admin_settings_fields._normalize_document_classification_categories) +// and refuses a bad list on save. These checks run while typing so each problem is shown +// on the row that has it, before Save is ever pressed. + +export interface ClassificationCategory { + label: string; + color: string; +} + +/** Mirrors DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH in admin_settings_fields.py. */ +export const CLASSIFICATION_LABEL_MAX_LENGTH = 80; + +/** The colour a new category starts with, matching the classic editor. */ +export const CLASSIFICATION_DEFAULT_COLOR = '#808080'; + +const HEX_COLOR = /^#[0-9a-fA-F]{6}$/; + +export function isHexColor(value: string): boolean { + return HEX_COLOR.test(value.trim()); +} + +/** + * A colour safe to paint with, or undefined. + * + * The stored list is administrator data from the database and may predate validation, so + * only a six-digit hex ever reaches a style; anything else falls back to the neutral badge. + */ +export function safeBadgeColor(value: string): string | undefined { + return isHexColor(value) ? value.trim() : undefined; +} + +/** Read the stored value defensively. */ +export function readClassificationCategories(value: unknown): ClassificationCategory[] { + if (!Array.isArray(value)) { + return []; + } + return value + .filter((item): item is Record => Boolean(item) && typeof item === 'object') + .map((item) => ({ + label: typeof item.label === 'string' ? item.label : '', + color: typeof item.color === 'string' ? item.color : '', + })); +} + +/** + * The problem with each row, or null when it would save. + * + * A repeated label is reported on the later row, naming the earlier one, because documents + * store the label itself: two categories with one label could never be told apart. + */ +export function describeCategoryProblems(categories: ClassificationCategory[]): (string | null)[] { + const firstUse = new Map(); + return categories.map((category, index) => { + const label = category.label.trim(); + if (!label) { + return 'Give this category a label.'; + } + if (label.length > CLASSIFICATION_LABEL_MAX_LENGTH) { + return `Keep the label to ${CLASSIFICATION_LABEL_MAX_LENGTH} characters.`; + } + const folded = label.toLocaleLowerCase(); + const earlier = firstUse.get(folded); + if (earlier !== undefined) { + return `Same label as category ${earlier + 1}. Each label must be unique.`; + } + firstUse.set(folded, index); + if (!isHexColor(category.color)) { + return 'Use a six-digit hex colour, such as #808080.'; + } + return null; + }); +} diff --git a/application/v2_ui/src/lib/retentionPolicy.ts b/application/v2_ui/src/lib/retentionPolicy.ts new file mode 100644 index 000000000..62c8c7151 --- /dev/null +++ b/application/v2_ui/src/lib/retentionPolicy.ts @@ -0,0 +1,406 @@ +// retentionPolicy.ts +// Pure helpers behind the Retention Policy controls in Admin Settings. +// +// Kept apart from the components so the decisions that have to be exactly right -- which +// workspace types a run may touch, whether unsaved edits block it, what an hour or a +// stored timestamp reads as, and how the server's response is summarised -- can be +// executed in a test instead of judged from a screenshot. +// +// Nothing here talks to the network. The components call the existing +// /api/admin/retention-policy/* routes and hand the responses to these functions. + +import { ApiError } from './apiClient'; +import { asBoolean } from './adminFields'; +import type { Json } from './types'; + +export type RetentionScope = 'personal' | 'group' | 'public'; + +export const RETENTION_SCOPES: readonly RetentionScope[] = ['personal', 'group', 'public']; + +export const RETENTION_SCOPE_LABELS: Readonly> = { + personal: 'Personal workspaces', + group: 'Group workspaces', + public: 'Public workspaces', +}; + +/** What a run's "affected" count is counted in, per type. */ +export const RETENTION_AFFECTED_NOUNS: Readonly> = { + personal: ['user', 'users'], + group: ['group', 'groups'], + public: ['workspace', 'workspaces'], +}; + +export const RETENTION_EXECUTION_HOUR_KEY = 'retention_policy_execution_hour'; +export const RETENTION_LAST_RUN_KEY = 'retention_policy_last_run'; +export const RETENTION_NEXT_RUN_KEY = 'retention_policy_next_run'; +export const CONVERSATION_ARCHIVING_KEY = 'enable_conversation_archiving'; + +/** Mirrors RETENTION_EXECUTION_HOUR_DEFAULT in admin_settings_fields.py. */ +export const RETENTION_EXECUTION_HOUR_DEFAULT = 2; + +export interface RetentionScopeKeys { + enabled: string; + conversation: string; + document: string; +} + +export function retentionScopeKeys(scope: RetentionScope): RetentionScopeKeys { + return { + enabled: `enable_retention_policy_${scope}`, + conversation: `default_retention_conversation_${scope}`, + document: `default_retention_document_${scope}`, + }; +} + +/** + * Every stored setting a reset depends on. + * + * An unsaved edit to any of these blocks Reset: the server works from what is saved, so + * running it beside a different, unsaved set of defaults would apply something other than + * what is on screen. + */ +export const RETENTION_SETTING_KEYS: readonly string[] = [ + ...RETENTION_SCOPES.flatMap((scope) => { + const keys = retentionScopeKeys(scope); + return [keys.enabled, keys.conversation, keys.document]; + }), + RETENTION_EXECUTION_HOUR_KEY, +]; + +/** A run also reads whether to archive conversations before removing them. */ +export const RETENTION_RUN_KEYS: readonly string[] = [ + ...RETENTION_SETTING_KEYS, + CONVERSATION_ARCHIVING_KEY, +]; + +/** Which of `keys` hold an unsaved edit. */ +export function unsavedKeys(draft: Json, keys: readonly string[]): string[] { + return keys.filter((key) => Object.prototype.hasOwnProperty.call(draft, key)); +} + +/** Workspace types whose retention is switched on in the saved settings. */ +export function savedEnabledScopes(settings: Json): RetentionScope[] { + return RETENTION_SCOPES.filter((scope) => asBoolean(settings[retentionScopeKeys(scope).enabled])); +} + +/** + * A retention period as people read it. + * + * Mirrors resolve_retention_value on the server: anything that is not a positive whole + * number of days means nothing is deleted. + */ +export function retentionPeriodLabel(value: unknown): string { + const days = readRetentionDays(value); + return days ? `${days} day${days === 1 ? '' : 's'}` : 'No automatic deletion'; +} + +/** "deleted after 30 days" or "kept", for a sentence about what a default does. */ +export function retentionDefaultPhrase(value: unknown): string { + const days = readRetentionDays(value); + return days ? `deleted after ${days} day${days === 1 ? '' : 's'}` : 'kept'; +} + +/** + * What a workspace type's saved defaults mean for a run or a reset, in one sentence. + * + * When both defaults keep everything the useful fact is who a run can still touch -- only + * those who chose their own period -- so that is what is said instead of "kept, kept". + */ +export function scopeDefaultsSummary( + settings: Json, + scope: RetentionScope, + action: 'run' | 'reset', +): string { + const keys = retentionScopeKeys(scope); + const conversation = retentionDefaultPhrase(settings[keys.conversation]); + const document = retentionDefaultPhrase(settings[keys.document]); + if (conversation === 'kept' && document === 'kept') { + return action === 'run' + ? `Defaults keep everything, so only ${RETENTION_AFFECTED_NOUNS[scope][1]} with their own period are affected.` + : 'Will follow defaults that keep everything.'; + } + const detail = `conversations ${conversation}, documents ${document}.`; + return action === 'run' ? `Defaults: ${detail}` : `Will follow: ${detail}`; +} + +function readRetentionDays(value: unknown): number | null { + const text = String(value ?? '').trim(); + if (!/^\d+$/.test(text)) { + return null; + } + const days = Number.parseInt(text, 10); + return days > 0 ? days : null; +} + +/** The saved hour, read defensively: settings written by hand may hold a string. */ +export function readExecutionHour(value: unknown): number { + const parsed = typeof value === 'number' ? value : Number.parseInt(String(value ?? ''), 10); + return Number.isInteger(parsed) && parsed >= 0 && parsed <= 23 + ? parsed + : RETENTION_EXECUTION_HOUR_DEFAULT; +} + +/** The keys whose saved change moves the next run. Mirrors RETENTION_SCHEDULE_KEYS. */ +export const RETENTION_SCHEDULE_KEYS: readonly string[] = [ + ...RETENTION_SCOPES.map((scope) => retentionScopeKeys(scope).enabled), + RETENTION_EXECUTION_HOUR_KEY, +]; + +/** + * Whether saving the draft will move the next run. + * + * Mirrors _apply_retention_schedule: a schedule key has to be in the save, and either its + * value differs from what is stored or no next run is stored yet. A switch flipped and + * flipped back is still in the draft but moves nothing. + */ +export function willReschedule(settings: Json, draft: Json): boolean { + const submitted = unsavedKeys(draft, RETENTION_SCHEDULE_KEYS); + if (!submitted.length) { + return false; + } + const changed = submitted.some((key) => draft[key] !== settings[key]); + return changed || !settings[RETENTION_NEXT_RUN_KEY]; +} + +/** + * The next run a save would schedule for `hour`, as an ISO timestamp. + * + * Mirrors compute_retention_next_run: today at that hour UTC, or tomorrow once it has + * passed. + */ +export function projectNextRun(hour: number, now: Date = new Date()): string { + const next = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, 0, 0, 0), + ); + if (next.getTime() <= now.getTime()) { + next.setUTCDate(next.getUTCDate() + 1); + } + return next.toISOString(); +} + +/** `02:00 UTC (2 AM)` -- the option wording for the run-time select. */ +export function utcHourLabel(hour: number): string { + const clock = `${String(hour).padStart(2, '0')}:00 UTC`; + const spoken = + hour === 0 ? 'midnight' : hour === 12 ? 'noon' : hour < 12 ? `${hour} AM` : `${hour - 12} PM`; + return `${clock} (${spoken})`; +} + +/** + * The run hour on the reader's own clock, or null when their clock already reads UTC. + * + * Worked out for today's date, so a daylight-saving change is reflected the day it happens. + * `timeZone` and `locale` exist for tests; the browser's own are used otherwise. + */ +export function localTimeForUtcHour( + hour: number, + now: Date = new Date(), + { locale, timeZone }: { locale?: string; timeZone?: string } = {}, +): string | null { + const instant = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, 0, 0), + ); + const parts = new Intl.DateTimeFormat('en-US', { + hour: 'numeric', + minute: 'numeric', + hourCycle: 'h23', + timeZone, + }).formatToParts(instant); + const localHour = Number(parts.find((part) => part.type === 'hour')?.value); + const localMinute = Number(parts.find((part) => part.type === 'minute')?.value); + if (localHour === hour && localMinute === 0) { + return null; + } + return new Intl.DateTimeFormat(locale, { + hour: 'numeric', + minute: '2-digit', + timeZoneName: 'short', + timeZone, + }).format(instant); +} + +/** Parse a stored ISO timestamp, tolerating Python's six fractional digits. */ +export function parseStoredTimestamp(value: unknown): Date | null { + const text = typeof value === 'string' ? value.trim() : ''; + if (!text) { + return null; + } + const parsed = new Date(text.replace(/(\.\d{3})\d+/, '$1')); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +/** "in 12 hours", "yesterday", "3 days ago". */ +export function relativeTimeLabel(date: Date, now: Date = new Date(), locale?: string): string { + const difference = date.getTime() - now.getTime(); + const distance = Math.abs(difference); + const minute = 60_000; + const hour = 60 * minute; + const day = 24 * hour; + if (distance < minute) { + return difference >= 0 ? 'in under a minute' : 'just now'; + } + const [amount, unit]: [number, Intl.RelativeTimeFormatUnit] = + distance < hour + ? [Math.round(distance / minute), 'minute'] + : distance < day + ? [Math.round(distance / hour), 'hour'] + : [Math.round(distance / day), 'day']; + return new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }).format( + difference >= 0 ? amount : -amount, + unit, + ); +} + +export interface RunTimeReadout { + /** The moment in UTC, which is what the schedule is set in. */ + absolute: string; + relative: string; + /** Whether the moment has already passed. */ + past: boolean; + iso: string; +} + +export function describeRunTime( + value: unknown, + now: Date = new Date(), + locale?: string, +): RunTimeReadout | null { + const date = parseStoredTimestamp(value); + if (!date) { + return null; + } + return { + absolute: new Intl.DateTimeFormat(locale, { + weekday: 'short', + month: 'short', + day: 'numeric', + year: 'numeric', + hour: 'numeric', + minute: '2-digit', + timeZone: 'UTC', + timeZoneName: 'short', + }).format(date), + relative: relativeTimeLabel(date, now, locale), + past: date.getTime() <= now.getTime(), + iso: date.toISOString(), + }; +} + +function readCount(value: unknown): number { + const parsed = typeof value === 'number' ? value : Number(value); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 0; +} + +function asRecord(value: unknown): Record { + return value && typeof value === 'object' ? (value as Record) : {}; +} + +function readScopes(value: unknown, fallback: readonly RetentionScope[]): RetentionScope[] { + if (!Array.isArray(value)) { + return [...fallback]; + } + const named = value.filter((item): item is RetentionScope => + (RETENTION_SCOPES as readonly unknown[]).includes(item), + ); + return RETENTION_SCOPES.filter((scope) => named.includes(scope)); +} + +export interface RetentionRunRow { + scope: RetentionScope; + conversations: number; + documents: number; + affected: number; +} + +export interface RetentionRunSummary { + ok: boolean; + rows: RetentionRunRow[]; + conversations: number; + documents: number; + /** + * How many problems the run reported. The server's text is not shown: it can be a raw + * exception message, which is for the logs rather than the browser. + */ + errorCount: number; +} + +/** Summarise `POST /api/admin/retention-policy/execute`. */ +export function summarizeRetentionRun( + response: unknown, + requested: readonly RetentionScope[], +): RetentionRunSummary { + const body = asRecord(response); + const results = asRecord(body.results); + const scopes = readScopes(results.scopes_processed, requested); + const rows = scopes.map((scope) => { + const counts = asRecord(results[scope]); + return { + scope, + conversations: readCount(counts.conversations), + documents: readCount(counts.documents), + affected: readCount(scope === 'personal' ? counts.users_affected : counts.workspaces_affected), + }; + }); + const errors = Array.isArray(results.errors) ? results.errors.length : 0; + return { + ok: body.success !== false && results.success !== false, + rows, + conversations: rows.reduce((total, row) => total + row.conversations, 0), + documents: rows.reduce((total, row) => total + row.documents, 0), + errorCount: errors, + }; +} + +export interface RetentionResetSummary { + ok: boolean; + rows: { scope: RetentionScope; updated: number }[]; + total: number; +} + +/** Summarise `POST /api/admin/retention-policy/force-push`. */ +export function summarizeRetentionReset( + response: unknown, + requested: readonly RetentionScope[], +): RetentionResetSummary { + const body = asRecord(response); + const details = asRecord(body.details); + const rows = readScopes(body.scopes, requested).map((scope) => ({ + scope, + updated: readCount(details[scope]), + })); + return { + ok: body.success !== false, + rows, + total: readCount(body.updated_count) || rows.reduce((total, row) => total + row.updated, 0), + }; +} + +/** + * What to say when a run or reset request fails. + * + * A 400 carries a validation message the route wrote for people, so it is shown. Anything + * else is described rather than quoted, because a 500 from these routes can carry raw + * exception text. A gateway timeout or a dropped connection gets its own wording: the work + * keeps going on the server after the browser stops waiting. + */ +export function retentionRequestError(error: unknown, action: 'run' | 'reset'): string { + if (error instanceof ApiError) { + if (error.status === 400) { + const payload = asRecord(error.payload); + if (typeof payload.error === 'string' && payload.error.trim()) { + return payload.error.trim(); + } + } + if (error.isAuthError) { + return 'Your session no longer has administrator access. Sign in again, then retry.'; + } + if (error.status !== 502 && error.status !== 503 && error.status !== 504) { + return action === 'run' + ? 'Retention could not finish. Check the application logs for the cause.' + : 'The reset could not finish. Check the application logs for the cause.'; + } + } + return action === 'run' + ? 'The page stopped waiting before retention reported back. It may still be running; check Last run in a few minutes.' + : 'The page stopped waiting before the reset reported back. Some workspaces may already follow the defaults.'; +} diff --git a/application/v2_ui/src/pages/AdminSettingsPage.tsx b/application/v2_ui/src/pages/AdminSettingsPage.tsx index f5354af7b..90045adae 100644 --- a/application/v2_ui/src/pages/AdminSettingsPage.tsx +++ b/application/v2_ui/src/pages/AdminSettingsPage.tsx @@ -39,6 +39,7 @@ import { BrandingImageField } from '../components/admin/BrandingImageField'; import { ChatDefaultModel } from '../components/admin/ChatDefaultModel'; import { CapabilityModelPicker } from '../components/admin/CapabilityModelPicker'; import { ChatModeNotice } from '../components/admin/ChatModeNotice'; +import { ClassificationCategoriesEditor } from '../components/admin/ClassificationCategoriesEditor'; import { ConnectionTest } from '../components/admin/ConnectionTest'; import { CustomPagesTable } from '../components/admin/CustomPagesTable'; import { EnhancedCitationsStorageTest } from '../components/admin/EnhancedCitationsStorageTest'; @@ -59,6 +60,8 @@ import { ModelSelectionPicker } from '../components/admin/ModelSelectionPicker'; import { OrchestrationCard } from '../components/admin/OrchestrationCard'; import { OrchestrationPlannerModelPicker } from '../components/admin/OrchestrationPlannerModelPicker'; import { PromotedAgentsEditor } from '../components/admin/PromotedAgentsEditor'; +import { RetentionResetDefaults, RetentionRunNow } from '../components/admin/RetentionOperations'; +import { RetentionSchedule } from '../components/admin/RetentionSchedule'; import { AgentDelegationManager } from '../components/agents/AgentDelegationManager'; import { GLOBAL_DELEGATION_SCOPE } from '../lib/agentDelegation'; import { SaveBar } from '../components/admin/SaveBar'; @@ -706,6 +709,18 @@ export function AdminSettingsPage() { const readSibling = (key: string, fallback = '') => asString(readFieldValue(READ_ONLY_REF(key), settings, draft), fallback); + /** + * Fold values the server changed outside a save into the page's copy of the settings. + * + * A retention run writes its own last and next run. Without this the schedule beside + * it would keep showing the times from when the page was opened. + */ + const mergeStoredSettings = useCallback((partial: Json) => { + setData((current) => + current ? { ...current, settings: { ...current.settings, ...partial } } : current, + ); + }, []); + /** * Move the page to a section, from a cross-reference elsewhere on it. * @@ -1079,6 +1094,53 @@ export function AdminSettingsPage() { help={field.help} /> ); + case 'retention-schedule': + return ( + field.key && setValue(field.key, next)} + onStoredSettingsChange={mergeStoredSettings} + /> + ); + case 'retention-reset-defaults': + return ( + + ); + case 'retention-run-now': + return ( + + ); + case 'document-classification-categories': + return ( + field.key && setValue(field.key, next)} + /> + ); default: return null; } diff --git a/docs/admin/data-lifecycle.md b/docs/admin/data-lifecycle.md index 943d29cff..ac9650a2c 100644 --- a/docs/admin/data-lifecycle.md +++ b/docs/admin/data-lifecycle.md @@ -1,7 +1,7 @@ --- layout: page title: "Data Lifecycle settings" -description: "Data Lifecycle groups retention, classification, and conversation archiving decisions." +description: "Data Lifecycle decides how long conversations and documents are kept, which labels a document can carry, and whether a deleted conversation can still be reviewed." section: "Administration" audience: admin admin_tab: data-lifecycle @@ -12,11 +12,20 @@ admin_tab: data-lifecycle ## What this group controls -Data Lifecycle groups retention, classification, and conversation archiving decisions. +Data Lifecycle decides three things about the data people create in SimpleChat: + +- **Retention**: how long conversations and documents are kept before they are deleted + automatically, set separately for personal, group and public workspaces. +- **Classification**: the labels a document can carry, such as *Confidential* or *Public*. +- **Archiving**: whether a deleted conversation is gone or kept for review. ## Why it matters -Lifecycle choices shape compliance and recovery expectations. Retention and archiving preserve evidence, while classification helps users and reviewers understand document sensitivity. +Retention is the only part of SimpleChat that deletes user data on a schedule without anyone +asking. Classification tells readers and reviewers how sensitive a document is before they open +it. Archiving decides whether a deletion, whether a user made it or retention did, can be +reviewed afterwards. Together they are your organisation's records position, so agree them +with whoever owns that policy before switching anything on. {% include media.html src="admin/data-lifecycle-overview.png" alt="Screenshot placeholder for the Data Lifecycle group in Admin Settings." title="Data Lifecycle settings" capture="Capture the Data Lifecycle group in Admin Settings showing its tabs." %} @@ -24,69 +33,180 @@ Lifecycle choices shape compliance and recovery expectations. Retention and arch ## Before you change anything -- Confirm retention requirements for every workspace scope. -- Agree on classification labels before enabling classification. -- Decide whether deleted conversations should be recoverable. +- Agree a period for each workspace type with your records owner. Personal drafts, group + records and organisation-wide knowledge rarely share one answer. +- Decide whether deletions must be reviewable before retention runs. Archiving only covers + deletions made while it is on. +- Agree the classification labels first. Renaming or removing a label later does not relabel + the documents that already carry it. ## Retention {#retention} ### Retention Policy {#retention-policy-section} -The Retention Policy section belongs to the Retention tab. Use it with the adjacent settings in this group so related rollout, access, and operational choices stay aligned. +Retention is switched on separately for each workspace type. Switching a type on lets that +type's owners choose how long their conversations and documents are kept; until they choose, +the organization defaults set here apply. **No automatic deletion** keeps everything, and it is +the starting default for every type. + +#### Who chooses the period + +| Workspace type | Who chooses | What the period governs | +| --- | --- | --- | +| Personal | Each user, from their own settings | Their documents, and their conversations other than chats grounded in a group or public workspace | +| Group | Group owners and admins | The group's documents, and every conversation grounded in the group, including members' own chats | +| Public | Workspace owners and admins | The workspace's documents, and every conversation grounded in it, including other users' chats with its documents | + +A conversation is *grounded* in its primary workspace, which is usually the workspace whose +documents it cited first. That workspace's policy governs the conversation, not its owner's +personal period. Public workspaces have no conversations of their own, so their conversation +period applies only to chats grounded in them. Deleting a public workspace leaves the chats +grounded in it in place, and they return to their owners' personal periods. + +If retention is off for a workspace type, nothing grounded in that type is deleted by +retention, including chats that would otherwise have followed an owner's personal period. + +#### What a run deletes + +- **Conversations** with no activity for longer than their period. With + [Conversation Archiving](#conversation-archiving-section) on, each is copied to the archive + before it is removed; with it off, the conversation and its messages are deleted permanently. +- **Documents** not updated for longer than their period. They are always deleted permanently, + together with their search index entries and stored files. +- **Notifications** follow every run. Users are told about their own items, group members + through their group, and public workspace managers receive a count of removed conversations. + The titles of chats grounded in a public workspace go only to each chat's owner, never to the + workspace's managers. + +#### When it runs + +Retention runs once a day at the chosen hour, in UTC. The new interface shows the hour on your +own clock beside it, with the last run and the next run. + +Saving a new hour, or switching a workspace type on, schedules the next run. Switching every +type off clears it. In the new interface, a save that touches neither leaves a pending run where +it is; the classic page reschedules on every save. The scheduler checks every five minutes, so a +run that is due starts within five minutes. + +#### Run now + +**Run retention now** deletes everything already past its period without waiting for the daily +run. It opens a review first: choose the workspace types to clean up, check the defaults each +one follows and what happens to conversations given the current archiving setting, then +confirm. The result reports what was removed for each type. + +In the new interface: + +- Only workspace types whose retention is on in the saved settings can be chosen. The classic + page's Manual Execution lets you pick a type whose retention is off, and runs it anyway. +- It waits while Retention or archiving changes are unsaved, because a run applies the saved + settings. Save or discard first. +- A run on a large deployment can take several minutes. If the page stops waiting before the + run reports back, the run carries on; use **Check again** beside **Last run** to see when it + finishes. + +#### Reset to the defaults + +**Reset to the defaults**, called *Force Push Defaults* on the classic page, clears the period +every user, group or public workspace of the chosen types has set, so each follows the +organization defaults again. Nothing is deleted by the reset itself; the next run applies the +defaults. It cannot be undone, because the periods people chose are not kept. + +The classic page saves the whole form before pushing. The new interface instead waits until +unsaved Retention changes are saved or discarded, and offers only types whose retention is on. #### Settings | Setting | What it does | Default | Notes | | --- | --- | --- | --- | -| Enable for Personal Workspaces | Exposes the capability after required services, permissions, and rollout policy are ready. | Off | `enable_retention_policy_personal`; capability toggle | -| Enable for Group Workspaces | Exposes the capability after required services, permissions, and rollout policy are ready. | Off | `enable_retention_policy_group`; capability toggle | -| Enable for Public Workspaces | Exposes the capability after required services, permissions, and rollout policy are ready. | Off | `enable_retention_policy_public`; capability toggle | -| Conversation Retention | Defines behavior for the related admin workflow; verify the affected feature after saving. | none | `default_retention_conversation_personal` | -| Document Retention | Defines behavior for the related admin workflow; verify the affected feature after saving. | none | `default_retention_document_personal` | -| Conversation Retention | Defines behavior for the related admin workflow; verify the affected feature after saving. | none | `default_retention_conversation_group` | -| Document Retention | Defines behavior for the related admin workflow; verify the affected feature after saving. | none | `default_retention_document_group` | -| Conversation Retention | Defines behavior for the related admin workflow; verify the affected feature after saving. | none | `default_retention_conversation_public` | -| Document Retention | Defines behavior for the related admin workflow; verify the affected feature after saving. | none | `default_retention_document_public` | -| Scheduled Execution Time (Hour of Day) | Retention policy will run once daily at this hour (UTC timezone). | 2 | `retention_policy_execution_hour` | +| Personal workspaces | Lets each user choose how long their own conversations and documents are kept. | Off | `enable_retention_policy_personal`; "Enable for Personal Workspaces" on the classic page | +| Group workspaces | Lets group owners and admins choose a period for the group and the chats grounded in it. | Off | `enable_retention_policy_group` | +| Public workspaces | Lets public workspace owners and admins choose a period for the workspace's documents and the chats grounded in it. | Off | `enable_retention_policy_public` | +| Default conversation retention (personal) | The conversation period for every user who has not chosen one. | No automatic deletion | `default_retention_conversation_personal` | +| Default document retention (personal) | The document period for every user who has not chosen one. | No automatic deletion | `default_retention_document_personal` | +| Default conversation retention (group) | The conversation period for every group that has not chosen one. | No automatic deletion | `default_retention_conversation_group` | +| Default document retention (group) | The document period for every group that has not chosen one. | No automatic deletion | `default_retention_document_group` | +| Default conversation retention (public) | The period for chats grounded in a public workspace that has not chosen one. | No automatic deletion | `default_retention_conversation_public` | +| Default document retention (public) | The document period for every public workspace that has not chosen one. | No automatic deletion | `default_retention_document_public` | +| Daily run time | The hour, in UTC, at which retention runs each day. | 02:00 UTC | `retention_policy_execution_hour`; a whole hour from 0 to 23. "Scheduled Execution Time" on the classic page | + +The defaults offer the same periods in both interfaces: no automatic deletion, 1 to 7 days, 10, +14, 21, 30, 60 and 90 days, six months, one year and two years. ## Classification {#classification} ### Document Classification {#document-classification-section} -The Document Classification section belongs to the Classification tab. Use it with the adjacent settings in this group so related rollout, access, and operational choices stay aligned. +Classification lets people label each document with one of a fixed list of categories. The +label shows as a coloured badge on the document, and on the conversations that cite it, so a +reader sees how sensitive the source is before relying on an answer. Documents can be filtered +by it. + +The categories are offered in the order you list them. In the new interface each category is a +row you edit in place: its label, its colour (picked or typed as a hex value), and buttons to +move or remove it. A preview shows the badges as users will see them. The section reads +**Needs configuration** while classification is on with no categories, because users would have +nothing to choose from. + +Documents store the label itself rather than a reference to the category. That is why: + +- two categories may not share a label, compared without regard to case; +- renaming or removing a category leaves documents that already carry the old label unchanged. #### Settings | Setting | What it does | Default | Notes | | --- | --- | --- | --- | -| Enable Document Classification | Exposes the capability after required services, permissions, and rollout policy are ready. | Off | `enable_document_classification`; capability toggle | +| Enable Document Classification | Lets users label documents with the categories below. | Off | `enable_document_classification`; capability toggle | +| Categories | The labels and badge colours users choose from, in display order. | None (grey), N/A (grey), Pending (blue) | `document_classification_categories`; labels up to 80 characters, colours as six-digit hex such as `#808080` | ## Archiving {#archiving} ### Conversation Archiving {#conversation-archiving-section} -The Conversation Archiving section belongs to the Archiving tab. Use it with the adjacent settings in this group so related rollout, access, and operational choices stay aligned. +With archiving on, a deleted conversation and its messages are copied to the archive before +they are removed, and a single deleted message is hidden rather than erased. Deleted messages no +longer appear in the conversation, in search results, or in the context sent to the model. +Archived copies are not shown to users; they are kept for compliance and audit review. + +Retention follows this setting too: conversations past their period are archived before they +are removed. Documents are always deleted permanently, whether archiving is on or off. #### Settings | Setting | What it does | Default | Notes | | --- | --- | --- | --- | -| Enable Conversation Archiving | Changes conversation deletion behavior to archive conversations instead of permanently deleting them immediately. Deleted chat messages are archived too, and no longer appear in the conversation, search results, or the context sent to the model. | Off | `enable_conversation_archiving`; capability toggle | +| Archive deleted conversations | Copies deleted conversations and messages to the archive instead of deleting them outright. | Off | `enable_conversation_archiving`; "Enable Conversation Archiving" on the classic page; capability toggle | ## Common tasks -1. **Apply retention policy.** Enable retention for intended scopes and check a test item after the policy job runs. Outcome to verify: Items follow the configured retention window. -2. **Classify documents.** Enable classification and upload a representative file. Outcome to verify: The document carries expected classification metadata. -3. **Archive conversations.** Enable archiving and delete a test conversation. Outcome to verify: Deletion follows the archive path. +1. **Keep group records for a year.** Switch **Group workspaces** on, set both group defaults + to *365 days (1 year)*, and save. Groups that never chose their own period follow it from + the next run. Outcome to verify: **Next run** shows a time. +2. **Bring every group back to the defaults.** Use **Reset to the defaults**, choose **Group + workspaces**, and confirm. Outcome to verify: the result reports how many groups were reset. +3. **Clean up now after tightening a default.** Save the new default, then use **Run retention + now**. Outcome to verify: the result lists what was removed and **Last run** updates. +4. **Add a sensitivity label.** Add a category, give it a unique label and a colour, and save. + Outcome to verify: the badge appears in the preview and in the document classification + choices. +5. **Make deletions reviewable.** Switch **Archive deleted conversations** on before retention + or users delete anything you need to keep. ## Troubleshooting | Symptom | Likely cause | Fix | | --- | --- | --- | -| Deleted conversations disappear permanently | Archiving was not enabled when deletion occurred. | Enable archiving before relying on recovery for future deletions. | +| **Run now** or **Reset to the defaults** cannot be clicked | There are unsaved Retention changes (or, for a run, an unsaved archiving change), or no workspace type has retention on in the saved settings. | Save or discard the changes, or switch a type on and save. | +| **Next run** says it is not set | Retention was switched on without a run time being saved, so no run has been scheduled yet. | Save the run time again, or let the next run set it. | +| A run removed nothing | No owner chose a period and the defaults keep everything, or nothing is older than its period yet. | Check the defaults shown in the review. | +| Chats grounded in a public workspace stopped following a user's personal period | Since 0.261.260 they follow the public workspace's period, as group-grounded chats follow their group's. | Switch **Public workspaces** on and set a default, or let workspace owners choose. | +| Saving categories fails because two categories share a label | Labels must be unique, without regard to case. | Rename one of them. | +| Deleted conversations disappear permanently | Archiving was off when the deletion happened. | Switch archiving on before relying on review of future deletions. | ## Related - [Administration settings overview]({{ '/admin/' | relative_url }}) - [Backup & Recovery settings]({{ '/admin/backup-recovery/' | relative_url }}) +- [Workspaces settings]({{ '/admin/workspaces/' | relative_url }}) - [Governance settings]({{ '/admin/governance/' | relative_url }}) diff --git a/docs/explanation/features/REACT_V2_UI.md b/docs/explanation/features/REACT_V2_UI.md index 81a884153..3a316cec5 100644 --- a/docs/explanation/features/REACT_V2_UI.md +++ b/docs/explanation/features/REACT_V2_UI.md @@ -941,6 +941,17 @@ The static page designer writes to the same `/api/admin/custom-pages` CRUD as th designer, so a page created in either interface is identical. Python-registered pages are listed but not editable, because they are defined in code. +#### Data Lifecycle group + +| Tab | What V2 now renders | +| --- | --- | +| Retention | A switch per workspace type, each with its two organization defaults nested beneath it while it is on; the daily run hour in UTC with the administrator's own clock, the last and next run; **Reset to the defaults** and **Run retention now** as inline reviews that use saved settings only, wait while retention edits are unsaved, and report per-type results | +| Classification | The capability switch and a category editor -- label, colour, hex, reorder, remove -- with per-row validation and a badge preview | +| Archiving | The archiving switch, with how retention follows it | + +Saving a workspace type or the run hour reschedules the next run as the classic form does. See +[V2 Admin Data Lifecycle Settings](V2_ADMIN_DATA_LIFECYCLE_SETTINGS.md). + #### Keeping the two interfaces in step The schema is a second description of settings the server-rendered panes also describe, so diff --git a/docs/explanation/features/V2_ADMIN_DATA_LIFECYCLE_SETTINGS.md b/docs/explanation/features/V2_ADMIN_DATA_LIFECYCLE_SETTINGS.md new file mode 100644 index 000000000..94d2654e0 --- /dev/null +++ b/docs/explanation/features/V2_ADMIN_DATA_LIFECYCLE_SETTINGS.md @@ -0,0 +1,150 @@ +# V2 Admin Data Lifecycle Settings + +**Implemented in version: 0.261.260** + +## Overview + +The V2 React admin surface renders its controls from the field schema in +`admin_settings_fields.py`. Until this change the Data Lifecycle group had no entry there, so +V2 drew it through the `enable_*` fallback scan: five bare switches with their storage keys +printed beneath them. Everything else the server-rendered panes offer was missing. + +This describes the group in full -- Retention Policy, Document Classification and +Conversation Archiving -- so V2 reaches parity with the classic page, in V2's own design +language. Two backend gaps found along the way are fixed at the same time; see +[Public Workspace Conversation Retention Fix](../fixes/PUBLIC_WORKSPACE_CONVERSATION_RETENTION_FIX.md). + +| Classic capability | V2 before | V2 now | +| --- | --- | --- | +| Per-type retention switches | Bare switches, keys printed | Switches with their scope icon and what they govern | +| Six organization defaults | Missing | Nested beneath their type's switch, shown only while it is on | +| Run hour (UTC) | Missing | 24-hour select, with the administrator's own clock beside it | +| Last and next run | Missing | Date, UTC time and relative time; the projected run while a change is unsaved | +| Manual Execution | Missing | **Run retention now**, an inline review with per-type results | +| Force Push Defaults | Missing | **Reset to the defaults**, an inline review with per-type counts | +| Classification categories | Missing | Row editor with colour, hex, reorder, remove, validation and a badge preview | +| Archiving | Bare switch | Switch with accurate help and the retention interplay | + +## Dependencies + +- `admin_settings_nav.py` -- the `data-lifecycle` group and its three section ids +- `route_backend_retention_policy.py` -- the existing execute, force-push and settings routes, + reused unchanged +- `functions_retention_policy.py` -- the retention job +- `components/documents/documentPresentation.tsx` -- `ClassificationBadge`, reused for the + category preview so it matches what users see + +## Technical specifications + +### Declared sections + +| Section | Fields | +| --- | --- | +| `retention-policy-section` | Three type switches, six default selects (each `depends_on` its type), `retention-schedule`, `retention-reset-defaults`, `retention-run-now` | +| `document-classification-section` | `enable_document_classification` (capability), `document-classification-categories` (component, `required`) | +| `conversation-archiving-section` | `enable_conversation_archiving` (capability, with an info notice) | + +The default selects mirror the classic pane's options value for value, which the parity test +checks. The schedule and both actions only appear while at least one type is on, through a +shared `RETENTION_ANY_SCOPE_ENABLED` dependency. + +### Component-backed keys + +Two values are saved through the settings PATCH but drawn by components, because no generic +control fits them: + +| Key | Why a component | Validation | +| --- | --- | --- | +| `retention_policy_execution_hour` | Stored as an int; `execute_retention_policy` passes it to `datetime.replace(hour=...)`, which raises on a string. A generic select saves strings. | Whole number 0 to 23, else refused | +| `document_classification_categories` | A list of `{label, color}` rows | Labels trimmed, non-empty, at most 80 characters, unique without regard to case; colours six-digit hex, saved lower-case | + +Both go through the new `_COMPONENT_VALUE_VALIDATORS` in `normalize_admin_settings_updates`. +It runs before the type-driven path, which refuses writes to a component, and unlike +`_DELEGATED_NORMALIZERS` it can refuse a value with a message. + +`LEGACY_FIELD_NAMES` maps `document_classification_categories` to the classic form's +`document_classification_categories_json` hidden field. + +### Rescheduling the next run + +The classic save handler recomputes `retention_policy_next_run` on every save. V2 now does the +equivalent in `_apply_retention_schedule`, applied once a save is known to be valid: + +- it runs only when a type switch or the hour is in the save; +- it moves the run only if one of those values changed, or no next run is stored; +- with every type off it clears the next run, otherwise it sets the next occurrence of the hour + in UTC through `compute_retention_next_run`, the same rule the classic handler and the job use. + +The derived key travels back in the PATCH response, so the readout updates without a reload. + +### Inline reviews instead of dialogs + +Run now and Reset open a review inside their own row rather than a modal: the workspace types +it may touch with the defaults each follows, plainly worded consequences that reflect the saved +archiving setting, then Cancel or confirm. Results replace the review in place. Keeping it +inline leaves the defaults above in view and does not hold the page behind a dialog during a +run that can take minutes. Focus moves to the review's heading when it opens and back to its +button when it closes. + +Both actions: + +- offer only the types whose retention is on in the **saved** settings; +- wait while an edit they depend on is unsaved -- any retention key for both, plus + `enable_conversation_archiving` for a run -- because the routes read the stored settings; +- never show the server's error text for a 500, which can carry a raw exception; a 400's + validation message is shown, and a gateway timeout is described as a run that may still be + in progress. + +After a run, the schedule is re-read from `GET /api/admin/retention-policy/settings` and merged +into the page through `mergeStoredSettings`, so **Last run** and **Next run** update. + +### APIs used + +| Route | Used by | +| --- | --- | +| `GET /api/v2/admin/settings`, `PATCH /api/v2/admin/settings` | Every declared field | +| `POST /api/admin/retention-policy/execute` | Run retention now | +| `POST /api/admin/retention-policy/force-push` | Reset to the defaults | +| `GET /api/admin/retention-policy/settings` | Schedule readout refresh | + +No route was added. All are `@admin_required`. + +### File structure + +| File | Role | +| --- | --- | +| `application/single_app/admin_settings_fields.py` | Section declarations, validators, `compute_retention_next_run`, `_apply_retention_schedule` | +| `application/single_app/functions_retention_policy.py` | Public workspace conversation retention | +| `application/v2_ui/src/lib/retentionPolicy.ts` | Pure helpers: held state, saved scopes, labels, timestamps, summaries, error wording | +| `application/v2_ui/src/lib/classificationCategories.ts` | Reading and checking categories | +| `application/v2_ui/src/components/admin/RetentionSchedule.tsx` | Run hour and schedule readout | +| `application/v2_ui/src/components/admin/RetentionOperations.tsx` | Run now and Reset reviews | +| `application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx` | Category editor and preview | +| `application/v2_ui/src/components/admin/retentionScopeIcons.ts` | One icon per workspace type | +| `application/v2_ui/src/components/admin/agentSectionAppearance.ts` | Scope icons on the switches; the Personal switch opted out of the derived primary emphasis, since the three types are peers | +| `application/v2_ui/src/pages/AdminSettingsPage.tsx` | Component branches and `mergeStoredSettings` | + +## Usage + +No configuration is required. Administrators find the controls at **Admin Settings > Data +Lifecycle** in the V2 interface. Behaviour, defaults and troubleshooting are described for +administrators in `docs/admin/data-lifecycle.md`. + +## Testing and validation + +| Test | Covers | +| --- | --- | +| `functional_tests/test_v2_admin_data_lifecycle_parity.py` | Panes match navigation; every classic field claimed; no invented fields; select values match classic; defaults match the application; hour and category normalization; next-run rescheduling against the classic rule; the routes the controls call exist and are admin-only; runs the TypeScript checks | +| `functional_tests/test_v2_admin_data_lifecycle_logic.ts` | Held states, saved scopes, labels, the local-clock hint, timestamps, run and reset summaries, error wording, category problems, and the rendered held and empty states | +| `functional_tests/test_retention_policy_conversation_scope_coverage.py` | Public rows in the ownership matrix, chats from deleted public workspaces returning to personal policy, public-grounded deletion and logging, notification privacy | +| `functional_tests/test_v2_admin_capability_placement.py` | Data Lifecycle receives no guessed fallback rows | +| `ui_tests/test_v2_admin_data_lifecycle.py` | Real controls instead of bare switches; reveal and reschedule on save; the hour saved as a number; run and reset reviews, results and focus; held while unsaved; failure wording without exception text; category editing, validation and ordered save; no overflow at 390px with large text in light and dark | + +### Known limitations + +- Run now is synchronous on the existing route. On a very large deployment the request can + outlast the gateway; the run continues, and the page says so. +- Category reordering uses row positions as keys, as the external links editor does, so focus + stays on the button that was pressed rather than following the moved row. +- At the largest text size on a phone, long words in the review can break mid-word, as they do + elsewhere on the page, rather than overflow the card. diff --git a/docs/explanation/fixes/PUBLIC_WORKSPACE_CONVERSATION_RETENTION_FIX.md b/docs/explanation/fixes/PUBLIC_WORKSPACE_CONVERSATION_RETENTION_FIX.md new file mode 100644 index 000000000..327218411 --- /dev/null +++ b/docs/explanation/fixes/PUBLIC_WORKSPACE_CONVERSATION_RETENTION_FIX.md @@ -0,0 +1,132 @@ +# Public Workspace Conversation Retention Fix + +## Header Information + +- **Fixed in version:** **0.261.260** +- **Area:** Retention policy for public workspaces, and retention scheduling from the V2 + admin surface +- **Related feature:** [V2 Admin Data Lifecycle Settings](../features/V2_ADMIN_DATA_LIFECYCLE_SETTINGS.md) + +### Issue Description + +Two retention problems surfaced while bringing the V2 Data Lifecycle settings to parity. + +**Public workspace conversation retention did nothing.** Both admin pages offer a default +conversation period for public workspaces, and public workspace owners can set their own, but +the retention job never applied either. `process_public_retention` skipped conversations +entirely, and the public branch of `_build_conversation_retention_sources` referred to +`cosmos_public_conversations_container` and `cosmos_public_messages_container`, which do not +exist; it would have raised `NameError` had it ever been reached. + +**Switching retention on from V2 did not schedule a run.** The classic save handler recomputes +`retention_policy_next_run` whenever it saves. The V2 PATCH saved the type switches through the +`enable_*` fallback without that step, so a type switched on in V2 left no next run. The +scheduler treats a missing next run as due, so retention ran within five minutes of the save +instead of at the configured hour. + +### Root Cause + +Public workspaces have no conversation store. A chat grounded in a public workspace is a +personal conversation in `conversations`, whose primary context is +`{type: primary, scope: public, id: }` and whose `chat_type` is `public`. The +0.250.103 ownership rework gave chats grounded in a group to the group's policy but left +public-grounded chats under their owner's personal policy, and the public pass was written as +though a separate public conversation container existed. + +The scheduling gap is a V1/V2 divergence: the rescheduling lived only in the server-rendered +form's save handler. + +## Technical Details + +### Policy Ownership Matrix + +| Conversation type | Storage | Governing policy | Activity field | +|---|---|---|---| +| Personal single-user | `conversations` | Creator's personal policy | `last_updated` | +| Group single-user | `conversations` | Primary group's policy | `last_updated` | +| **Public single-user** | `conversations` | **Primary public workspace's policy** | `last_updated` | +| Personal multi-user | `collaboration_conversations` | Creator's personal policy | `updated_at` | +| Group multi-user | `collaboration_conversations` | `scope.group_id` policy | `updated_at` | +| Legacy group | `group_conversations` | Group policy | `last_updated` | + +A public single-user chat is recognised by the workspace id in its primary context, not by +`chat_type` alone, and it follows the workspace's policy only while the workspace exists. +Deleting a public workspace removes only the workspace record, so its grounded chats stay in +`conversations` where no public run will reach them again. A chat with no workspace id, or +grounded in a workspace that no longer exists, therefore stays under its owner's personal policy +rather than falling outside every policy. + +The personal run lists the existing public workspaces once (`SELECT c.id`) and passes the set to +each user's pass. A failed or interrupted listing yields `None`, never an empty set: the +personal run then skips every grounded chat for that run instead of reading the failure as +every workspace having been deleted and removing chats under the wrong policy. + +### Files Modified + +| File | Change | +|---|---| +| `application/single_app/functions_retention_policy.py` | `_get_primary_public_workspace_id`, `_is_governed_by_public_workspace`, `_get_existing_public_workspace_ids`, `_build_public_scope_query`; personal pass excludes chats grounded in an existing public workspace; public source reads `conversations` and `messages`; `process_public_retention` deletes aged public-grounded chats; activity logs carry the grounded workspace id; owner notifications | +| `application/single_app/admin_settings_fields.py` | `compute_retention_next_run` and `_apply_retention_schedule`, applied to valid V2 saves; the run hour validated and stored as an int | +| `functional_tests/test_retention_policy_conversation_scope_coverage.py` | Public matrix rows, chats from deleted workspaces returning to personal policy, failed-listing safety, public deletion and logging, notification privacy | +| `functional_tests/test_v2_admin_data_lifecycle_parity.py` | Rescheduling against the classic rule, hour normalization | +| `application/single_app/config.py` | Version `0.261.260` | + +### Notification Privacy + +A public workspace's retention notification goes to its owner, admins and document managers. +Group notifications have always listed the titles of deleted chats, but a public workspace's +chats belong to arbitrary users across the organisation, and their titles are not the +workspace managers' to see. So: + +- the workspace notification reports how many conversations were removed, without titles, and + lists removed documents as before; +- each chat's owner receives a personal notification naming their own removed chats and the + workspace they were grounded in; +- records another process had already deleted are left out of owner notifications, because + retention did not remove them. + +### Scheduling Rule + +`_apply_retention_schedule` runs only when a type switch or the run hour is in a valid save. It +moves the run when one of those values changed, or when no next run is stored. With every type +off it clears the next run; otherwise it sets the next occurrence of the hour in UTC, the rule +`execute_retention_policy` and the classic handler use. An unrelated save never moves a pending +run. + +### Impact + +- **Behaviour change:** chats grounded in a public workspace now follow that workspace's + conversation period instead of their owner's personal period. Where public retention is off, + retention no longer deletes them, exactly as with group-grounded chats when group retention is + off. Administrators relying on personal retention to clear these chats should switch public + retention on and set a default. +- Public workspace owners' conversation period, already offered in both interfaces, now takes + effect. +- Switching retention on from V2 schedules the next run at the configured hour. +- **Known limitation, unchanged:** chats grounded in a group that has since been deleted are + still excluded from personal retention and reached by no group run. This change does not + alter group ownership. + +## Validation + +### Test Results + +- `functional_tests/test_retention_policy_conversation_scope_coverage.py`: the policy matrix + (including chats from deleted workspaces and a failed listing), the end-to-end personal run + over live, deleted and ungrounded workspaces, public-grounded deletion and logging, and + notification privacy pass. Three collaboration cleanup tests in the same file fail + identically before and after this change, because the harness cannot import + `functions_saved_analysis`; they are unrelated to retention scope. +- `functional_tests/test_v2_admin_data_lifecycle_parity.py`: 10 of 10 pass, including + rescheduling compared with the classic rule across hours and dates. + +### Before and After + +| Scenario | Before | After | +|---|---|---| +| Public workspace conversation period | Stored, never applied | Applied to chats grounded in the workspace | +| Chat grounded in a public workspace | Followed the owner's personal period | Follows the public workspace's period | +| Chat grounded in a deleted public workspace | Followed the owner's personal period | Still follows the owner's personal period | +| Public retention notification | No conversation section | Count only for managers; titles to each owner | +| Retention type switched on from V2 | No next run; ran within five minutes | Next run at the configured hour | +| Run hour saved from V2 | Not editable in V2 | Saved as an int; out-of-range values refused | diff --git a/functional_tests/test_retention_policy_conversation_scope_coverage.py b/functional_tests/test_retention_policy_conversation_scope_coverage.py index 96d20b5cd..3b230d714 100644 --- a/functional_tests/test_retention_policy_conversation_scope_coverage.py +++ b/functional_tests/test_retention_policy_conversation_scope_coverage.py @@ -1,16 +1,20 @@ # test_retention_policy_conversation_scope_coverage.py """ Functional test for retention policy conversation scope coverage. -Version: 0.261.038 +Version: 0.261.260 Implemented in: 0.250.103 +Public workspace conversation retention added in: 0.261.260 This test verifies the retention ownership matrix, timestamp safeguards, -collaboration cleanup, archival behavior, race handling, and new-group defaults. +collaboration cleanup, archival behavior, race handling, new-group defaults, and +that chats grounded in a public workspace follow that workspace's policy while it +exists, return to their owner's policy once it is deleted, and never reach anyone +but their owner by title. """ import ast import copy -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import os import sys import types @@ -141,8 +145,6 @@ def build_retention_namespace(): 'cosmos_group_conversations_container', 'cosmos_group_messages_container', 'cosmos_collaboration_conversations_container', - 'cosmos_public_conversations_container', - 'cosmos_public_messages_container', ) namespace = { 'datetime': datetime, @@ -159,9 +161,12 @@ def build_retention_namespace(): '_parse_retention_timestamp', '_get_primary_group_id', '_is_group_single_user_conversation', + '_get_primary_public_workspace_id', + '_is_governed_by_public_workspace', '_is_converted_conversation_source', '_is_aged_conversation', '_build_group_scope_query', + '_build_public_scope_query', '_build_conversation_retention_sources', }, assignment_names={ @@ -235,6 +240,92 @@ def test_policy_matrix_and_retention_values(): assert group_sources['group_multi_user']['matches_scope'](group_multi) assert group_sources['legacy_group']['matches_scope'](legacy_group) + # A chat grounded in a public workspace is a personal conversation that follows + # the workspace's policy, the way a group-grounded chat follows its group's. + public_sources = namespace['_build_conversation_retention_sources']( + 'public', + cutoff.isoformat(), + public_workspace_id='public-1', + ) + assert [source['name'] for source in public_sources] == ['public_single_user'] + public_source = public_sources[0] + assert public_source['container'] is namespace['cosmos_conversations_container'] + assert public_source['messages_container'] is namespace['cosmos_messages_container'] + assert "context_item.scope = 'public'" in public_source['query'] + assert {'name': '@scope_id', 'value': 'public-1'} in public_source['parameters'] + + public_single = { + 'id': 'public-single', + 'user_id': 'user-1', + 'chat_type': 'public', + 'last_updated': '2025-01-01T00:00:00Z', + 'context': [ + {'type': 'primary', 'scope': 'public', 'id': 'public-1'}, + {'type': 'secondary', 'scope': 'personal', 'id': 'user-1'}, + ], + } + other_public = copy.deepcopy(public_single) + other_public['context'][0]['id'] = 'public-2' + assert public_source['matches_scope'](public_single) + assert not public_source['matches_scope'](other_public) + assert not public_source['matches_scope'](personal_single) + assert not public_source['matches_scope'](group_single) + assert not personal_sources['personal_single_user']['matches_scope'](public_single) + assert not group_sources['group_single_user']['matches_scope'](public_single) + + converted_public = dict(public_single) + converted_public['collaboration_conversation_id'] = 'personal-multi' + assert not public_source['matches_scope'](converted_public) + + # A grounded chat with no workspace id stays under its owner's personal policy rather + # than falling outside every policy. + unresolved_public = { + 'id': 'unresolved-public', + 'user_id': 'user-1', + 'chat_type': 'public', + 'last_updated': '2025-01-01T00:00:00Z', + 'context': [{'type': 'primary', 'scope': 'public', 'id': ''}], + } + assert personal_sources['personal_single_user']['matches_scope'](unresolved_public) + assert not public_source['matches_scope'](unresolved_public) + + # A grounded chat follows its workspace only while the workspace exists. Deleting the + # workspace returns the chat to its owner's personal policy, since no public run will + # ever reach it again. When the workspaces could not be listed, the personal run + # leaves every grounded chat alone rather than risk the wrong policy. + orphaned_public = copy.deepcopy(public_single) + orphaned_public['id'] = 'orphaned-public' + orphaned_public['context'][0]['id'] = 'public-deleted' + + def personal_single_source(existing_public_workspace_ids): + return next( + source + for source in namespace['_build_conversation_retention_sources']( + 'personal', + cutoff.isoformat(), + user_id='user-1', + existing_public_workspace_ids=existing_public_workspace_ids, + ) + if source['name'] == 'personal_single_user' + ) + + listed = personal_single_source(frozenset({'public-1'})) + assert not listed['matches_scope'](public_single) + assert listed['matches_scope'](orphaned_public) + assert listed['matches_scope'](personal_single) + assert listed['matches_scope'](unresolved_public) + assert not listed['matches_scope'](group_single) + assert not public_source['matches_scope'](orphaned_public) + + every_workspace_deleted = personal_single_source(frozenset()) + assert every_workspace_deleted['matches_scope'](public_single) + + listing_failed = personal_single_source(None) + assert not listing_failed['matches_scope'](public_single) + assert not listing_failed['matches_scope'](orphaned_public) + assert listing_failed['matches_scope'](personal_single) + assert listing_failed['matches_scope'](unresolved_public) + converted_personal = dict(personal_single) converted_personal['collaboration_conversation_id'] = 'personal-multi' converted_group = dict(legacy_group) @@ -263,9 +354,11 @@ def test_policy_matrix_and_retention_values(): settings = { 'default_retention_conversation_personal': '30', 'default_retention_conversation_group': '90', + 'default_retention_conversation_public': '60', } assert resolve_value('default', 'personal', 'conversation', settings) == 30 assert resolve_value(None, 'group', 'conversation', settings) == 90 + assert resolve_value('default', 'public', 'conversation', settings) == 60 assert resolve_value('7', 'personal', 'conversation', settings) == 7 assert resolve_value('none', 'group', 'conversation', settings) == 'none' return True @@ -670,6 +763,414 @@ def test_new_groups_persist_default_retention_values(): return True +def test_public_grounded_deletion_logs_the_governing_workspace(): + """A public-grounded chat is archived, logged against its workspace, and owned.""" + conversation_item = { + 'id': 'public-chat-1', + 'user_id': 'owner-a', + 'title': 'Benefits questions', + 'chat_type': 'public', + 'last_updated': '2025-01-01T00:00:00Z', + 'context': [{'type': 'primary', 'scope': 'public', 'id': 'public-1'}], + 'tags': [], + } + message_item = { + 'id': 'public-chat-1-message-1', + 'conversation_id': 'public-chat-1', + 'role': 'user', + 'content': 'What is the leave policy?', + } + conversations = FakeContainer([conversation_item]) + messages = FakeContainer([message_item]) + archived_conversations = FakeContainer() + archived_messages = FakeContainer() + effects = {'archival_logs': [], 'deletion_logs': [], 'archived_thoughts': []} + + namespace = { + 'datetime': datetime, + 'timezone': timezone, + 'CosmosResourceNotFoundError': FakeCosmosResourceNotFoundError, + 'cosmos_conversations_container': conversations, + 'cosmos_messages_container': messages, + 'cosmos_group_conversations_container': FakeContainer(), + 'cosmos_group_messages_container': FakeContainer(), + 'cosmos_collaboration_conversations_container': FakeContainer(), + 'cosmos_archived_conversations_container': archived_conversations, + 'cosmos_archived_messages_container': archived_messages, + 'cancel_m365_conversation_deliveries': lambda conversation_id: None, + 'log_conversation_archival': ( + lambda **kwargs: effects['archival_logs'].append(copy.deepcopy(kwargs)) + ), + 'log_conversation_deletion': ( + lambda **kwargs: effects['deletion_logs'].append(copy.deepcopy(kwargs)) + ), + 'delete_blob_backed_chat_message_files': lambda *args, **kwargs: None, + 'archive_thoughts_for_conversation': ( + lambda conversation_id, user_id, **kwargs: effects['archived_thoughts'].append( + (conversation_id, user_id) + ) + ), + 'delete_thoughts_for_conversation': lambda *args, **kwargs: None, + 'invalidate_conversation_cache_for_item': lambda item, reason: None, + 'debug_print': lambda *args, **kwargs: None, + } + loaded = load_source_members( + RETENTION_FILE, + { + '_parse_retention_timestamp', + '_get_primary_group_id', + '_is_group_single_user_conversation', + '_get_primary_public_workspace_id', + '_is_governed_by_public_workspace', + '_is_converted_conversation_source', + '_is_aged_conversation', + '_build_group_scope_query', + '_build_public_scope_query', + '_build_conversation_retention_sources', + '_delete_standard_conversation_for_retention', + }, + assignment_names={ + 'GROUP_SINGLE_USER_CHAT_TYPES', + 'PERSONAL_MULTI_USER_CHAT_TYPE', + 'GROUP_MULTI_USER_CHAT_TYPE', + }, + namespace=namespace, + ) + + cutoff = datetime(2026, 1, 1, tzinfo=timezone.utc) + source = loaded['_build_conversation_retention_sources']( + 'public', + cutoff.isoformat(), + public_workspace_id='public-1', + )[0] + detail = loaded['_delete_standard_conversation_for_retention']( + copy.deepcopy(conversation_item), + source, + 'public', + True, + cutoff, + ) + + assert detail['id'] == 'public-chat-1' + assert detail['user_id'] == 'owner-a' + assert 'public-chat-1' not in conversations.items + assert message_item['id'] not in messages.items + archived = archived_conversations.items['public-chat-1'] + assert archived['retention_source'] == 'public_single_user' + assert archived['archived_by_retention_policy'] is True + assert message_item['id'] in archived_messages.items + assert effects['archived_thoughts'] == [('public-chat-1', 'owner-a')] + assert effects['archival_logs'][0]['public_workspace_id'] == 'public-1' + assert effects['deletion_logs'][0]['public_workspace_id'] == 'public-1' + assert effects['deletion_logs'][0]['workspace_type'] == 'public' + return True + + +def test_public_workspace_run_keeps_chat_titles_with_their_owners(): + """The workspace hears a count; each owner hears only about their own chats.""" + calls = { + 'conversations': [], + 'documents': [], + 'workspace_notifications': [], + 'user_notifications': [], + } + + def fake_delete_aged_conversations(**kwargs): + calls['conversations'].append(kwargs) + return { + 'count': 3, + 'details': [ + {'id': 'chat-1', 'title': 'Budget draft', 'user_id': 'owner-a'}, + {'id': 'chat-2', 'title': 'Hiring notes', 'user_id': 'owner-b'}, + { + 'id': 'chat-3', + 'title': 'Removed elsewhere', + 'user_id': 'owner-a', + 'already_deleted': True, + }, + ], + } + + def fake_delete_aged_documents(**kwargs): + calls['documents'].append(kwargs) + return {'count': 0, 'details': []} + + namespace = { + 'datetime': datetime, + 'timezone': timezone, + 'logging': __import__('logging'), + 'get_all_public_workspaces': lambda: [ + { + 'id': 'public-1', + 'name': 'HR Policies', + 'retention_policy': { + 'conversation_retention_days': 'default', + 'document_retention_days': 'none', + }, + }, + { + 'id': 'public-2', + 'name': 'Kept forever', + 'retention_policy': { + 'conversation_retention_days': 'none', + 'document_retention_days': 'none', + }, + }, + ], + 'get_settings': lambda: {'default_retention_conversation_public': '30'}, + 'delete_aged_conversations': fake_delete_aged_conversations, + 'delete_aged_documents': fake_delete_aged_documents, + 'send_retention_notification': ( + lambda workspace_id, summary, workspace_type: calls[ + 'workspace_notifications' + ].append((workspace_id, copy.deepcopy(summary), workspace_type)) + ), + 'create_notification': ( + lambda **kwargs: calls['user_notifications'].append(copy.deepcopy(kwargs)) + ), + 'log_event': lambda *args, **kwargs: None, + 'debug_print': lambda *args, **kwargs: None, + } + loaded = load_source_members( + RETENTION_FILE, + { + 'resolve_retention_value', + 'process_public_retention', + 'send_public_conversation_owner_notifications', + }, + namespace=namespace, + ) + + results = loaded['process_public_retention']() + + assert calls['conversations'] == [{ + 'public_workspace_id': 'public-1', + 'retention_days': 30, + 'workspace_type': 'public', + }] + assert calls['documents'] == [] + assert results['conversations'] == 3 + assert results['workspaces_affected'] == 1 + + assert len(calls['workspace_notifications']) == 1 + workspace_id, summary, workspace_type = calls['workspace_notifications'][0] + assert (workspace_id, workspace_type) == ('public-1', 'public') + assert summary['conversations_deleted'] == 3 + assert summary['conversation_details'] == [], ( + "The workspace's managers must not be sent the titles of other users' chats." + ) + + by_owner = { + notification['user_id']: notification + for notification in calls['user_notifications'] + } + assert set(by_owner) == {'owner-a', 'owner-b'} + owner_a = by_owner['owner-a']['message'] + owner_b = by_owner['owner-b']['message'] + assert 'Budget draft' in owner_a and 'Hiring notes' not in owner_a + assert 'Hiring notes' in owner_b and 'Budget draft' not in owner_b + assert 'Removed elsewhere' not in owner_a + assert 'HR Policies' in owner_a + assert by_owner['owner-a']['metadata']['conversations_deleted'] == 1 + assert by_owner['owner-a']['metadata']['public_workspace_id'] == 'public-1' + return True + + +def test_public_workspace_listing_never_reads_a_failure_as_no_workspaces(): + """A failed or partial listing is None, never a set that would release grounded chats.""" + + class ListingContainer: + def __init__(self, rows): + self.rows = rows + self.queries = [] + + def query_items(self, query=None, enable_cross_partition_query=False, **kwargs): + self.queries.append((query, enable_cross_partition_query)) + return self.rows() + + def list_workspace_ids(rows): + container = ListingContainer(rows) + loaded = load_source_members( + RETENTION_FILE, + {'_get_existing_public_workspace_ids'}, + namespace={ + 'cosmos_public_workspaces_container': container, + 'logging': __import__('logging'), + 'log_event': lambda *args, **kwargs: None, + 'debug_print': lambda *args, **kwargs: None, + }, + ) + return loaded['_get_existing_public_workspace_ids'](), container.queries + + def refused(): + raise RuntimeError('throttled') + + def interrupted(): + yield {'id': 'public-1'} + raise RuntimeError('continuation failed') + + workspace_ids, queries = list_workspace_ids( + lambda: iter([{'id': 'public-1'}, {'id': ' public-2 '}, {'id': ''}, {}]) + ) + assert workspace_ids == frozenset({'public-1', 'public-2'}) + assert queries == [('SELECT c.id FROM c', True)] + assert list_workspace_ids(lambda: iter([]))[0] == frozenset() + assert list_workspace_ids(refused)[0] is None + assert list_workspace_ids(interrupted)[0] is None, ( + "A partial listing would hand chats in unlisted workspaces to personal retention." + ) + return True + + +def test_personal_run_takes_back_chats_from_deleted_public_workspaces(): + """A chat whose public workspace is gone follows its owner's policy again.""" + + def aged_chat(conversation_id, public_workspace_id=None): + chat = { + 'id': conversation_id, + 'user_id': 'owner-a', + 'title': conversation_id, + 'chat_type': 'public' if public_workspace_id else 'personal_single_user', + 'last_updated': '2000-01-01T00:00:00Z', + 'tags': [], + } + if public_workspace_id: + chat['context'] = [ + {'type': 'primary', 'scope': 'public', 'id': public_workspace_id}, + ] + return chat + + def run_personal_retention(existing_public_workspace_ids): + conversations = FakeContainer([ + aged_chat('plain'), + aged_chat('live-grounded', 'public-live'), + aged_chat('orphaned', 'public-deleted'), + ]) + deletion_logs = [] + namespace = { + 'datetime': datetime, + 'timezone': timezone, + 'timedelta': timedelta, + 'CosmosResourceNotFoundError': FakeCosmosResourceNotFoundError, + 'cosmos_conversations_container': conversations, + 'cosmos_messages_container': FakeContainer(), + 'cosmos_group_conversations_container': FakeContainer(), + 'cosmos_group_messages_container': FakeContainer(), + 'cosmos_collaboration_conversations_container': FakeContainer(), + 'get_settings': lambda: {'enable_conversation_archiving': False}, + 'cancel_m365_conversation_deliveries': lambda conversation_id: None, + 'log_conversation_deletion': ( + lambda **kwargs: deletion_logs.append(copy.deepcopy(kwargs)) + ), + 'delete_blob_backed_chat_message_files': lambda *args, **kwargs: None, + 'delete_thoughts_for_conversation': lambda *args, **kwargs: None, + 'invalidate_conversation_cache_for_item': lambda item, reason: None, + 'log_event': lambda *args, **kwargs: None, + 'debug_print': lambda *args, **kwargs: None, + } + loaded = load_source_members( + RETENTION_FILE, + { + '_parse_retention_timestamp', + '_get_primary_group_id', + '_is_group_single_user_conversation', + '_get_primary_public_workspace_id', + '_is_governed_by_public_workspace', + '_is_converted_conversation_source', + '_is_aged_conversation', + '_build_group_scope_query', + '_build_public_scope_query', + '_build_conversation_retention_sources', + '_delete_standard_conversation_for_retention', + 'delete_aged_conversations', + }, + assignment_names={ + 'GROUP_SINGLE_USER_CHAT_TYPES', + 'PERSONAL_MULTI_USER_CHAT_TYPE', + 'GROUP_MULTI_USER_CHAT_TYPE', + }, + namespace=namespace, + ) + result = loaded['delete_aged_conversations']( + 30, + workspace_type='personal', + user_id='owner-a', + existing_public_workspace_ids=existing_public_workspace_ids, + ) + return result, conversations, deletion_logs + + result, conversations, deletion_logs = run_personal_retention( + frozenset({'public-live'}) + ) + assert sorted(detail['id'] for detail in result['details']) == ['orphaned', 'plain'] + assert set(conversations.items) == {'live-grounded'}, ( + "A chat grounded in a live public workspace must wait for that workspace's policy." + ) + orphan_log = next( + log for log in deletion_logs if log['conversation_id'] == 'orphaned' + ) + assert orphan_log['workspace_type'] == 'personal' + assert orphan_log['public_workspace_id'] == 'public-deleted' + assert orphan_log['additional_context']['retention_source'] == 'personal_single_user' + + # Without a listing, grounded chats are left for a later run. + result, conversations, _ = run_personal_retention(None) + assert [detail['id'] for detail in result['details']] == ['plain'] + assert set(conversations.items) == {'live-grounded', 'orphaned'} + return True + + +def test_personal_run_lists_public_workspaces_once_for_every_user(): + """One listing per run reaches every user's conversation pass.""" + listings = [] + conversation_calls = [] + + def list_public_workspaces(): + listings.append(True) + return frozenset({'public-live'}) + + def fake_delete_aged_conversations(**kwargs): + conversation_calls.append(kwargs) + return {'count': 0, 'details': []} + + def user(user_id, conversation_retention_days): + return { + 'id': user_id, + 'settings': { + 'retention_policy': { + 'conversation_retention_days': conversation_retention_days, + 'document_retention_days': 'none', + }, + }, + } + + namespace = { + 'get_all_user_settings': lambda: [user('owner-a', '30'), user('owner-b', 'default')], + 'get_settings': lambda: {'default_retention_conversation_personal': '60'}, + '_get_existing_public_workspace_ids': list_public_workspaces, + 'delete_aged_conversations': fake_delete_aged_conversations, + 'log_event': lambda *args, **kwargs: None, + 'debug_print': lambda *args, **kwargs: None, + } + loaded = load_source_members( + RETENTION_FILE, + {'resolve_retention_value', 'process_personal_retention'}, + namespace=namespace, + ) + + loaded['process_personal_retention']() + + assert len(listings) == 1 + assert [ + (call['user_id'], call['retention_days'], call['existing_public_workspace_ids']) + for call in conversation_calls + ] == [ + ('owner-a', 30, frozenset({'public-live'})), + ('owner-b', 60, frozenset({'public-live'})), + ] + return True + + if __name__ == '__main__': tests = [ test_policy_matrix_and_retention_values, @@ -677,6 +1178,11 @@ def test_new_groups_persist_default_retention_values(): test_collaboration_archival_covers_group_source_records, test_collaboration_revalidation_and_cleanup_failure_safety, test_new_groups_persist_default_retention_values, + test_public_grounded_deletion_logs_the_governing_workspace, + test_public_workspace_run_keeps_chat_titles_with_their_owners, + test_public_workspace_listing_never_reads_a_failure_as_no_workspaces, + test_personal_run_takes_back_chats_from_deleted_public_workspaces, + test_personal_run_lists_public_workspaces_once_for_every_user, ] results = [] for test in tests: diff --git a/functional_tests/test_v2_admin_capability_placement.py b/functional_tests/test_v2_admin_capability_placement.py index 8c45da801..b5f1b4657 100644 --- a/functional_tests/test_v2_admin_capability_placement.py +++ b/functional_tests/test_v2_admin_capability_placement.py @@ -2,8 +2,9 @@ #!/usr/bin/env python3 """ Functional test pinning where the V2 admin surface files each capability toggle. -Version: 0.261.122 +Version: 0.261.260 Implemented in: 0.261.047 +Data Lifecycle added to the fully described groups in: 0.261.260 Settings that ``admin_settings_fields.py`` does not describe are still shown in the V2 admin UI, by scanning the settings document for ``enable_*`` booleans and @@ -68,7 +69,13 @@ # Groups whose sections are described by the schema in full. A guessed row landing # in one of these is a key that was filed by word stems into a group that has a # real home for everything it owns, which means it is in the wrong place. -FULLY_DESCRIBED_GROUP_IDS = (APPEARANCE_GROUP_ID, "chat", "security", "agents-actions") +FULLY_DESCRIBED_GROUP_IDS = ( + APPEARANCE_GROUP_ID, + "chat", + "security", + "agents-actions", + "data-lifecycle", +) # Where each relocated toggle now lives, and the V1 pane it is mirrored from. The # pane is checked too, because a schema field with no server-rendered counterpart diff --git a/functional_tests/test_v2_admin_data_lifecycle_logic.ts b/functional_tests/test_v2_admin_data_lifecycle_logic.ts new file mode 100644 index 000000000..082fa81b7 --- /dev/null +++ b/functional_tests/test_v2_admin_data_lifecycle_logic.ts @@ -0,0 +1,382 @@ +// test_v2_admin_data_lifecycle_logic.ts +// +// Runtime checks for the V2 Admin Settings Data Lifecycle controls. +// Version: 0.261.260 +// Implemented in: 0.261.260 +// +// The decisions behind Retention Policy and Document Classification are easy to get +// subtly wrong and invisible in a screenshot: whether an unsaved edit holds a run, which +// workspace types a run may touch, whether a save will move the next run, what an hour or +// a stored timestamp reads as, how a run's response is summarised, and which category row +// a problem belongs to. They are executed here, and the components are rendered to markup +// to prove the held and empty states actually reach the page. +// +// Run by test_v2_admin_data_lifecycle_parity.py, which bundles this with the esbuild Vite +// already brings in and executes it under node, skipping when the front-end toolchain is +// absent. + +import assert from 'node:assert/strict'; +import { createElement } from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { ApiError } from '../application/v2_ui/src/lib/apiClient'; +import { + CONVERSATION_ARCHIVING_KEY, + RETENTION_RUN_KEYS, + RETENTION_SCHEDULE_KEYS, + RETENTION_SETTING_KEYS, + describeRunTime, + localTimeForUtcHour, + parseStoredTimestamp, + projectNextRun, + readExecutionHour, + retentionDefaultPhrase, + retentionPeriodLabel, + retentionRequestError, + savedEnabledScopes, + scopeDefaultsSummary, + summarizeRetentionReset, + summarizeRetentionRun, + unsavedKeys, + utcHourLabel, + willReschedule, +} from '../application/v2_ui/src/lib/retentionPolicy'; +import { + describeCategoryProblems, + readClassificationCategories, + safeBadgeColor, +} from '../application/v2_ui/src/lib/classificationCategories'; +import { + RetentionResetDefaults, + RetentionRunNow, +} from '../application/v2_ui/src/components/admin/RetentionOperations'; +import { RetentionSchedule } from '../application/v2_ui/src/components/admin/RetentionSchedule'; +import { ClassificationCategoriesEditor } from '../application/v2_ui/src/components/admin/ClassificationCategoriesEditor'; +import type { AdminField } from '../application/v2_ui/src/lib/adminFields'; + +const checks: [string, () => void][] = []; +function check(name: string, fn: () => void) { + checks.push([name, fn]); +} + +const NOW = new Date('2026-10-06T17:43:00Z'); + +check('periods read the way resolve_retention_value applies them', () => { + assert.equal(retentionPeriodLabel('none'), 'No automatic deletion'); + assert.equal(retentionPeriodLabel('30'), '30 days'); + assert.equal(retentionPeriodLabel(1), '1 day'); + for (const nothing of ['0', '', 'abc', null, undefined, -5, '7.5']) { + assert.equal(retentionPeriodLabel(nothing), 'No automatic deletion', String(nothing)); + } + assert.equal(retentionDefaultPhrase('90'), 'deleted after 90 days'); + assert.equal(retentionDefaultPhrase('none'), 'kept'); +}); + +check('a review says what a type defaults mean, and who a run can touch when they keep everything', () => { + const settings = { + default_retention_conversation_group: '90', + default_retention_document_group: '365', + default_retention_conversation_personal: 'none', + default_retention_document_personal: 'none', + }; + assert.equal( + scopeDefaultsSummary(settings, 'group', 'run'), + 'Defaults: conversations deleted after 90 days, documents deleted after 365 days.', + ); + assert.equal( + scopeDefaultsSummary(settings, 'group', 'reset'), + 'Will follow: conversations deleted after 90 days, documents deleted after 365 days.', + ); + assert.equal( + scopeDefaultsSummary(settings, 'personal', 'run'), + 'Defaults keep everything, so only users with their own period are affected.', + ); + assert.equal(scopeDefaultsSummary(settings, 'personal', 'reset'), 'Will follow defaults that keep everything.'); + assert.match(scopeDefaultsSummary({}, 'public', 'run'), /only workspaces with their own period/); +}); + +check('the run hour is read defensively and labelled in UTC', () => { + assert.equal(readExecutionHour(5), 5); + assert.equal(readExecutionHour('7'), 7); + assert.equal(readExecutionHour(24), 2); + assert.equal(readExecutionHour('soon'), 2); + assert.equal(utcHourLabel(0), '00:00 UTC (midnight)'); + assert.equal(utcHourLabel(2), '02:00 UTC (2 AM)'); + assert.equal(utcHourLabel(12), '12:00 UTC (noon)'); + assert.equal(utcHourLabel(13), '13:00 UTC (1 PM)'); +}); + +check('the local-clock hint is omitted on UTC and shown everywhere else', () => { + assert.equal(localTimeForUtcHour(2, NOW, { locale: 'en-US', timeZone: 'UTC' }), null); + const eastern = localTimeForUtcHour(2, NOW, { locale: 'en-US', timeZone: 'America/New_York' }); + assert.ok(eastern && /10:00\s?PM/.test(eastern), String(eastern)); + const india = localTimeForUtcHour(2, NOW, { locale: 'en-US', timeZone: 'Asia/Kolkata' }); + assert.ok(india && /7:30\s?AM/.test(india), String(india)); +}); + +check('a projected next run follows compute_retention_next_run', () => { + assert.equal( + projectNextRun(2, new Date('2026-10-06T01:00:00Z')), + '2026-10-06T02:00:00.000Z', + ); + assert.equal( + projectNextRun(2, new Date('2026-10-06T02:00:00Z')), + '2026-10-07T02:00:00.000Z', + ); + assert.equal(projectNextRun(23, NOW), '2026-10-06T23:00:00.000Z'); +}); + +check('only a changed schedule key, or a missing next run, moves the next run', () => { + const stored = { + enable_retention_policy_group: true, + retention_policy_execution_hour: 2, + retention_policy_next_run: '2026-10-07T02:00:00+00:00', + }; + assert.equal(willReschedule(stored, {}), false); + assert.equal(willReschedule(stored, { default_retention_document_group: '30' }), false); + assert.equal(willReschedule(stored, { retention_policy_execution_hour: 5 }), true); + assert.equal(willReschedule(stored, { enable_retention_policy_personal: true }), true); + // Flipped and flipped back: still in the draft, but nothing moves. + assert.equal(willReschedule(stored, { enable_retention_policy_group: true }), false); + assert.equal( + willReschedule({ ...stored, retention_policy_next_run: null }, { retention_policy_execution_hour: 2 }), + true, + ); + assert.deepEqual( + [...RETENTION_SCHEDULE_KEYS].sort(), + [ + 'enable_retention_policy_group', + 'enable_retention_policy_personal', + 'enable_retention_policy_public', + 'retention_policy_execution_hour', + ], + ); +}); + +check('stored timestamps tolerate Python microseconds and read relative to now', () => { + const parsed = parseStoredTimestamp('2026-10-07T02:00:00.123456+00:00'); + assert.ok(parsed); + assert.equal(parsed?.toISOString(), '2026-10-07T02:00:00.123Z'); + assert.equal(parseStoredTimestamp('not a time'), null); + assert.equal(parseStoredTimestamp(null), null); + + const next = describeRunTime('2026-10-07T02:00:00+00:00', NOW, 'en-US'); + assert.ok(next); + assert.equal(next?.past, false); + assert.match(next?.relative ?? '', /in 8 hours/); + assert.match(next?.absolute ?? '', /UTC/); + + const last = describeRunTime('2026-10-05T02:00:00+00:00', NOW, 'en-US'); + assert.equal(last?.past, true); + assert.match(last?.relative ?? '', /2 days ago/); +}); + +check('runs and resets read only saved settings, and the right edits hold them', () => { + const settings = { + enable_retention_policy_personal: false, + enable_retention_policy_group: true, + enable_retention_policy_public: 'true', + }; + assert.deepEqual(savedEnabledScopes(settings), ['group', 'public']); + assert.ok(RETENTION_RUN_KEYS.includes(CONVERSATION_ARCHIVING_KEY)); + assert.ok(!RETENTION_SETTING_KEYS.includes(CONVERSATION_ARCHIVING_KEY)); + assert.deepEqual(unsavedKeys({ enable_conversation_archiving: true }, RETENTION_SETTING_KEYS), []); + assert.deepEqual( + unsavedKeys({ enable_conversation_archiving: true }, RETENTION_RUN_KEYS), + ['enable_conversation_archiving'], + ); + assert.deepEqual(unsavedKeys({ app_title: 'x' }, RETENTION_RUN_KEYS), []); +}); + +check('a run response is summarised per processed type, without its error text', () => { + const summary = summarizeRetentionRun( + { + success: true, + results: { + success: true, + scopes_processed: ['public', 'personal'], + personal: { conversations: 4, documents: 2, users_affected: 3, details: [] }, + group: { conversations: 9, documents: 9, workspaces_affected: 9 }, + public: { conversations: 1, documents: 5, workspaces_affected: 2 }, + errors: [], + }, + }, + ['personal', 'public'], + ); + assert.equal(summary.ok, true); + assert.deepEqual(summary.rows.map((row) => row.scope), ['personal', 'public']); + assert.deepEqual(summary.rows[0], { scope: 'personal', conversations: 4, documents: 2, affected: 3 }); + assert.equal(summary.rows[1].affected, 2); + assert.equal(summary.conversations, 5); + assert.equal(summary.documents, 7); + assert.equal(summary.errorCount, 0); + + const failed = summarizeRetentionRun( + { success: false, results: { success: false, errors: ['Traceback: secret detail'] } }, + ['group'], + ); + assert.equal(failed.ok, false); + assert.equal(failed.errorCount, 1); + assert.deepEqual(failed.rows, [{ scope: 'group', conversations: 0, documents: 0, affected: 0 }]); + assert.ok(!JSON.stringify(failed).includes('secret detail')); +}); + +check('a reset response is summarised per type', () => { + const summary = summarizeRetentionReset( + { success: true, updated_count: 12, scopes: ['group', 'personal'], details: { personal: 10, group: 2 } }, + ['personal', 'group'], + ); + assert.deepEqual(summary.rows, [ + { scope: 'personal', updated: 10 }, + { scope: 'group', updated: 2 }, + ]); + assert.equal(summary.total, 12); + const withoutTotal = summarizeRetentionReset({ success: true, details: { public: 3 } }, ['public']); + assert.equal(withoutTotal.total, 3); +}); + +check('request failures quote only messages written for people', () => { + const invalid = new ApiError('Bad request', 400, { success: false, error: 'Invalid workspace scopes: tenant' }); + assert.equal(retentionRequestError(invalid, 'run'), 'Invalid workspace scopes: tenant'); + const crashed = new ApiError('x', 500, { error: 'Failed to execute retention policy: KeyError secret' }); + const crashedMessage = retentionRequestError(crashed, 'run'); + assert.ok(!crashedMessage.includes('KeyError')); + assert.match(crashedMessage, /application logs/); + assert.match(retentionRequestError(new ApiError('x', 504, ''), 'run'), /may still be running/); + assert.match(retentionRequestError(new TypeError('Failed to fetch'), 'run'), /may still be running/); + assert.match(retentionRequestError(new ApiError('x', 403, {}), 'reset'), /administrator access/); +}); + +check('category problems land on the row that has them', () => { + const problems = describeCategoryProblems([ + { label: 'Public', color: '#00ff00' }, + { label: ' ', color: '#808080' }, + { label: 'public', color: '#123456' }, + { label: 'Secret', color: 'red' }, + { label: 'x'.repeat(81), color: '#808080' }, + { label: 'Secret', color: '#808080' }, + ]); + assert.equal(problems[0], null); + assert.match(problems[1] ?? '', /label/); + assert.match(problems[2] ?? '', /Same label as category 1/); + assert.match(problems[3] ?? '', /hex/); + assert.match(problems[4] ?? '', /80 characters/); + // A row with a bad colour still claims its label. + assert.match(problems[5] ?? '', /Same label as category 4/); +}); + +check('stored categories are read defensively and only hex colours are painted', () => { + assert.deepEqual(readClassificationCategories('nope'), []); + assert.deepEqual( + readClassificationCategories([{ label: 'A', color: '#000000' }, null, 'x', { label: 3 }]), + [{ label: 'A', color: '#000000' }, { label: '', color: '' }], + ); + assert.equal(safeBadgeColor('#0000FF'), '#0000FF'); + assert.equal(safeBadgeColor('red; background:url(x)'), undefined); +}); + +const field = (overrides: Partial): AdminField => ({ + type: 'component', + label: 'Label', + help: 'Help text.', + ...overrides, +}); + +const savedSettings = { + enable_retention_policy_personal: false, + enable_retention_policy_group: true, + enable_retention_policy_public: false, + default_retention_conversation_group: '30', + default_retention_document_group: 'none', + enable_conversation_archiving: true, + retention_policy_execution_hour: 2, +}; + +check('run now is offered on saved settings and held by an unsaved edit', () => { + const render = (draft: Record) => + renderToStaticMarkup( + createElement(RetentionRunNow, { + field: field({ component: 'retention-run-now', label: 'Run retention now' }), + settings: savedSettings, + draft, + onStoredSettingsChange: () => undefined, + onOpenSection: () => undefined, + }), + ); + const ready = render({}); + assert.match(ready, /Run now…/); + assert.doesNotMatch(ready, /Save or discard/); + assert.doesNotMatch(ready, /disabled=""/); + + for (const draft of [{ default_retention_document_group: '30' }, { enable_conversation_archiving: false }]) { + const held = render(draft); + assert.match(held, /Save or discard your changes first/); + assert.match(held, /disabled=""/); + } +}); + +check('reset is held by retention edits but not by archiving', () => { + const render = (draft: Record) => + renderToStaticMarkup( + createElement(RetentionResetDefaults, { + field: field({ component: 'retention-reset-defaults', label: 'Reset to the defaults' }), + settings: savedSettings, + draft, + }), + ); + assert.doesNotMatch(render({ enable_conversation_archiving: false }), /Save or discard/); + assert.match(render({ retention_policy_execution_hour: 5 }), /Save or discard your changes first/); + assert.match(render({}), /Reset to defaults…/); +}); + +check('the schedule shows the stored runs and every UTC hour', () => { + const markup = renderToStaticMarkup( + createElement(RetentionSchedule, { + field: field({ key: 'retention_policy_execution_hour', component: 'retention-schedule', label: 'Daily run time' }), + value: 2, + settings: { ...savedSettings, retention_policy_next_run: '2099-01-01T02:00:00+00:00' }, + draft: {}, + onChange: () => undefined, + onStoredSettingsChange: () => undefined, + }), + ); + assert.match(markup, /Last run/); + assert.match(markup, /Never run/); + assert.match(markup, /Next run/); + assert.match(markup, /2099/); + assert.equal((markup.match(/