diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index bdd25b97a..7ceeefbf9 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -83,8 +83,8 @@ the switch on without naming the companion, so an API client gets the same result. A value named in the same save always wins. -The Appearance, Agents & Actions, Chat, Knowledge, Operations, Workflow, Workspaces -and Security groups are described in full. Sections with no entry here fall back to the V2 surface's +The Appearance, Agents & Actions, Chat, Data Lifecycle, Knowledge, Operations, Workflow, +Workspaces and Security groups are described in full. Sections with no entry here fall back to the V2 surface's ``enable_*`` scan, so undescribed groups keep working exactly as they did. A handful of individual fields outside those groups are also declared: that scan places a key by guessing from shared word stems, and declaring a field is the @@ -113,6 +113,7 @@ import copy import json import re +from datetime import datetime, timedelta, timezone from urllib.parse import urlparse from admin_settings_secret_utils import ( @@ -409,6 +410,68 @@ ACCESS_DENIED_MESSAGE_MAX_LENGTH = 2000 +# The periods the server-rendered Retention pane offers for each organization +# default, in its order and with its wording. Stored as strings because that is what +# the classic form posts and what resolve_retention_value reads. The Data Lifecycle +# parity test compares these with the pane option for option. +RETENTION_PERIOD_OPTIONS = [ + {"value": "none", "label": "No automatic deletion"}, + {"value": "1", "label": "1 day"}, + {"value": "2", "label": "2 days"}, + {"value": "3", "label": "3 days"}, + {"value": "4", "label": "4 days"}, + {"value": "5", "label": "5 days"}, + {"value": "6", "label": "6 days"}, + {"value": "7", "label": "7 days (1 week)"}, + {"value": "10", "label": "10 days"}, + {"value": "14", "label": "14 days (2 weeks)"}, + {"value": "21", "label": "21 days (3 weeks)"}, + {"value": "30", "label": "30 days"}, + {"value": "60", "label": "60 days"}, + {"value": "90", "label": "90 days (3 months)"}, + {"value": "180", "label": "180 days (6 months)"}, + {"value": "365", "label": "365 days (1 year)"}, + {"value": "730", "label": "730 days (2 years)"}, +] + +# Workspace types retention can be switched on for, in display order. +RETENTION_SCOPES = ("personal", "group", "public") + +# Mirrors 'retention_policy_execution_hour' in functions_settings.py. The hour is UTC +# and must be stored as an int: execute_retention_policy hands it straight to +# datetime.replace(hour=...), which raises on a string and stops every run. +RETENTION_EXECUTION_HOUR_DEFAULT = 2 +RETENTION_EXECUTION_HOUR_MIN = 0 +RETENTION_EXECUTION_HOUR_MAX = 23 + +# Saving any of these reschedules the next retention run, as the classic form does. +RETENTION_SCHEDULE_KEYS = ( + "enable_retention_policy_personal", + "enable_retention_policy_group", + "enable_retention_policy_public", + "retention_policy_execution_hour", +) + +# The schedule and the run and reset actions only mean something while at least one +# workspace type has retention on. +RETENTION_ANY_SCOPE_ENABLED = { + "any_of": [ + {"key": f"enable_retention_policy_{scope}", "equals": True} + for scope in RETENTION_SCOPES + ] +} + +# A category is shown as a badge on documents and conversations, so its label is kept +# to badge length. The classic form sets no bound; this matches an external link label. +DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH = 80 + +# Mirrors the 'document_classification_categories' default in functions_settings.py. +DOCUMENT_CLASSIFICATION_DEFAULT_CATEGORIES = [ + {"label": "None", "color": "#808080"}, + {"label": "N/A", "color": "#808080"}, + {"label": "Pending", "color": "#0000FF"}, +] + ADMIN_SETTINGS_FIELDS = { "branding-section": [ @@ -6056,6 +6119,206 @@ "depends_on": {"key": "enable_custom_rate_limit_message", "equals": True}, }, ], + # ------------------------------------------------------------------ + # Data Lifecycle + # + # What happens to user data over time: retention deletes it, classification + # labels it, and archiving decides whether a deletion can still be reviewed. + # + # Retention has no single switch. Each workspace type is switched on by itself + # and carries its own pair of organization defaults, which the server-rendered + # pane only reveals once that type is on; declaring the defaults as dependents + # of their type's switch gives V2 the same reveal and nests them beneath it. + # + # The run hour and the two actions are components. The hour is stored as an int, + # which a generic select cannot produce, and the actions call + # /api/admin/retention-policy/* rather than saving a value. + # ------------------------------------------------------------------ + "retention-policy-section": [ + { + "key": "enable_retention_policy_personal", + "type": "switch", + "label": "Personal workspaces", + "help": ( + "Each user chooses how long their own conversations and documents " + "are kept. Conversations idle and documents unchanged for longer " + "than that are deleted. Until a user chooses, the defaults below " + "apply." + ), + "default": False, + }, + { + "key": "default_retention_conversation_personal", + "type": "select", + "label": "Default conversation retention", + "help": "Applies to every user who has not chosen their own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_personal", "equals": True}, + }, + { + "key": "default_retention_document_personal", + "type": "select", + "label": "Default document retention", + "help": "Applies to every user who has not chosen their own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_personal", "equals": True}, + }, + { + "key": "enable_retention_policy_group", + "type": "switch", + "label": "Group workspaces", + "help": ( + "Group owners and admins choose how long the group's conversations " + "and documents are kept, including members' own chats grounded in " + "the group. Until they choose, the defaults below apply." + ), + "default": False, + }, + { + "key": "default_retention_conversation_group", + "type": "select", + "label": "Default conversation retention", + "help": "Applies to every group that has not chosen its own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_group", "equals": True}, + }, + { + "key": "default_retention_document_group", + "type": "select", + "label": "Default document retention", + "help": "Applies to every group that has not chosen its own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_group", "equals": True}, + }, + { + "key": "enable_retention_policy_public", + "type": "switch", + "label": "Public workspaces", + "help": ( + "Workspace owners and admins choose how long the workspace's " + "documents are kept, and the conversations grounded in it. Until " + "they choose, the defaults below apply." + ), + "default": False, + }, + { + "key": "default_retention_conversation_public", + "type": "select", + "label": "Default conversation retention", + "help": ( + "Applies to every public workspace that has not chosen its own " + "period. A workspace's conversation period covers every chat " + "grounded in it, including other users' chats with its documents, " + "in place of their owner's personal period." + ), + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_public", "equals": True}, + }, + { + "key": "default_retention_document_public", + "type": "select", + "label": "Default document retention", + "help": "Applies to every public workspace that has not chosen its own period.", + "default": "none", + "options": RETENTION_PERIOD_OPTIONS, + "depends_on": {"key": "enable_retention_policy_public", "equals": True}, + }, + { + # Shows the stored last and next run beside the hour, and writes the hour + # into the draft as an int. Validated on save by + # _COMPONENT_VALUE_VALIDATORS. + "key": "retention_policy_execution_hour", + "type": "component", + "component": "retention-schedule", + "label": "Daily run time", + "help": ( + "Retention runs once a day at this hour, in UTC. Saving a new hour, " + "or switching a workspace type on, reschedules the next run." + ), + "default": RETENTION_EXECUTION_HOUR_DEFAULT, + "depends_on": RETENTION_ANY_SCOPE_ENABLED, + }, + { + "type": "component", + "component": "retention-reset-defaults", + "label": "Reset to the defaults", + "help": ( + "Clears the period every user, group or public workspace of the " + "chosen types has set, so each follows the organization defaults " + "above. It cannot be undone; they would have to choose again." + ), + "depends_on": RETENTION_ANY_SCOPE_ENABLED, + }, + { + "type": "component", + "component": "retention-run-now", + "label": "Run retention now", + "help": ( + "Deletes everything already past its period straight away instead " + "of waiting for the daily run. Owners are told what was removed, " + "as they are after a scheduled run." + ), + "depends_on": RETENTION_ANY_SCOPE_ENABLED, + }, + ], + "document-classification-section": [ + { + "key": "enable_document_classification", + "type": "switch", + "role": "capability", + "label": "Enable document classification", + "help": ( + "Users label each document with one of the categories below. The " + "label shows as a coloured badge on the document and on the " + "conversations that cite it, and documents can be filtered by it." + ), + "default": False, + }, + { + # Writes document_classification_categories into the draft. Validated on + # save by _COMPONENT_VALUE_VALIDATORS. Required, so a section switched on + # with no categories reads as needing configuration: users would have + # nothing to choose from. + "key": "document_classification_categories", + "type": "component", + "component": "document-classification-categories", + "label": "Categories", + "help": ( + "Offered in this order wherever a user picks one. Renaming or " + "removing a category does not change documents already labelled " + "with it." + ), + "default": DOCUMENT_CLASSIFICATION_DEFAULT_CATEGORIES, + "required": True, + "depends_on": {"key": "enable_document_classification", "equals": True}, + }, + ], + "conversation-archiving-section": [ + { + "key": "enable_conversation_archiving", + "type": "switch", + "role": "capability", + "label": "Archive deleted conversations", + "help": ( + "A deleted conversation and its messages are copied to the archive " + "before they are removed, and a single deleted message is hidden " + "rather than erased. Archived copies are not shown to users; they " + "are kept for compliance and audit review." + ), + "default": False, + "notice": ( + "Retention follows this setting too: conversations past their period " + "are archived before they are removed. Documents are always deleted " + "permanently." + ), + "notice_level": "info", + }, + ], "cosmos-maintenance-section": [ { "key": "enable_app_maintenance", @@ -6932,6 +7195,9 @@ "inbound_mcp_allowed_client_app_entries": ["inbound_mcp_allowed_client_app_entries_json"], "inbound_mcp_allowed_tenant_entries": ["inbound_mcp_allowed_tenant_entries_json"], "inbound_mcp_allowed_source_entries": ["inbound_mcp_allowed_source_entries_json"], + # Same again for the classification categories, whose table script writes the + # list into a hidden JSON field; V2 edits the stored list directly. + "document_classification_categories": ["document_classification_categories_json"], # V1 names the auto-turnoff switches after the control rather than after the # stored key, which reads the other way round. "debug_logging_timer_enabled": ["enable_debug_logging_timer"], @@ -7831,6 +8097,139 @@ def merged(key): ) +def _validate_retention_execution_hour(value): + """Return ``(hour, error)`` for the daily retention run hour. + + The hour is stored as an int, because execute_retention_policy hands it to + ``datetime.replace(hour=...)``, which raises on a string. The classic form quietly + falls back to 2 AM for an out-of-range hour; refusing the value says so instead. + """ + message = ( + f"Choose an hour between {RETENTION_EXECUTION_HOUR_MIN} and " + f"{RETENTION_EXECUTION_HOUR_MAX}." + ) + if isinstance(value, bool): + return None, message + if isinstance(value, int): + hour = value + elif isinstance(value, float) and value.is_integer(): + hour = int(value) + elif isinstance(value, str) and re.fullmatch(r"\s*\d{1,2}\s*", value): + hour = int(value) + else: + return None, message + if not RETENTION_EXECUTION_HOUR_MIN <= hour <= RETENTION_EXECUTION_HOUR_MAX: + return None, message + return hour, None + + +def _normalize_document_classification_categories(value): + """Return ``(categories, error)`` for the document classification list. + + Applies the checks the classic save handler does -- a list of objects, each with + a non-empty label and a colour -- and tightens two of them. The colour must be a + six-digit hex, because it is drawn as the badge background. Two categories may + not share a label, because a document stores the label rather than an id, so + once applied the two could never be told apart. + """ + if not isinstance(value, list): + return None, "Expected a list of categories." + + categories = [] + first_use = {} + for index, item in enumerate(value, start=1): + if not isinstance(item, dict): + return None, f"Category {index} is not an object." + + raw_label = item.get("label") + label = raw_label.strip() if isinstance(raw_label, str) else "" + if not label: + return None, f"Category {index} needs a label." + if len(label) > DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH: + return None, ( + f"Category {index}'s label is longer than " + f"{DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH} characters." + ) + + raw_color = item.get("color") + color = raw_color.strip() if isinstance(raw_color, str) else "" + if not HEX_COLOR_PATTERN.match(color): + return None, f"Category {index} needs a hex colour such as #808080." + + folded = label.casefold() + if folded in first_use: + return None, ( + f'Categories {first_use[folded]} and {index} are both labelled ' + f'"{label}". Each label must be unique.' + ) + first_use[folded] = index + categories.append({"label": label, "color": color.lower()}) + + return categories, None + + +# Component-backed keys that are saved through the settings PATCH. A component +# normally owns its own persistence, so the type-driven path refuses a write to one; +# these are the exceptions. Each validator returns ``(value, error)``, which is what +# lets it refuse a value with a message -- the delegated normalizers below return a +# value only and have no way to say no. +_COMPONENT_VALUE_VALIDATORS = { + "retention_policy_execution_hour": _validate_retention_execution_hour, + "document_classification_categories": _normalize_document_classification_categories, +} + + +def compute_retention_next_run(hour, now=None): + """Return the next retention run at ``hour`` UTC, as an ISO timestamp. + + The rule the classic save handler and execute_retention_policy both apply: today + at that hour, or tomorrow once the hour has passed. + """ + current = now or datetime.now(timezone.utc) + next_run = current.replace(hour=hour, minute=0, second=0, microsecond=0) + if next_run <= current: + next_run += timedelta(days=1) + return next_run.isoformat() + + +def _apply_retention_schedule(normalized, current_settings, now=None): + """Reschedule the next retention run when a save changes what it depends on. + + The classic form recomputes ``retention_policy_next_run`` on every save. Without the + same step, a workspace type switched on here leaves no next run at all, and the + scheduler treats retention as due and runs it within minutes rather than at the + chosen hour. Only a save that changes a scope switch or the hour reschedules, so an + unrelated save never moves a pending run; a missing next run is filled in whenever + one of those keys is saved. + """ + submitted = [key for key in RETENTION_SCHEDULE_KEYS if key in normalized] + if not submitted: + return + + changed = any(normalized[key] != current_settings.get(key) for key in submitted) + if not changed and current_settings.get("retention_policy_next_run"): + return + + def merged(key, fallback): + if key in normalized: + return normalized[key] + stored = current_settings.get(key) + return fallback if stored is None else stored + + enabled = any( + _coerce_bool(merged(f"enable_retention_policy_{scope}", False)) + for scope in RETENTION_SCOPES + ) + if not enabled: + normalized["retention_policy_next_run"] = None + return + + hour, hour_error = _validate_retention_execution_hour( + merged("retention_policy_execution_hour", RETENTION_EXECUTION_HOUR_DEFAULT) + ) + if hour_error: + hour = RETENTION_EXECUTION_HOUR_DEFAULT + normalized["retention_policy_next_run"] = compute_retention_next_run(hour, now) # Keys whose normalization already exists elsewhere. Reusing those functions is @@ -8299,6 +8698,16 @@ def normalize_admin_settings_updates(updates, current_settings=None): normalized[key] = _DELEGATED_NORMALIZERS[key](value, field) continue + # Component-backed keys that are saved here, validated by a function that + # can refuse a value rather than coerce it. + if key in _COMPONENT_VALUE_VALIDATORS: + validated, validation_error = _COMPONENT_VALUE_VALIDATORS[key](value) + if validation_error: + errors[key] = validation_error + else: + normalized[key] = validated + continue + if field is None: normalized[key] = value continue @@ -8393,6 +8802,9 @@ def normalize_admin_settings_updates(updates, current_settings=None): _apply_nested_paths(normalized, current) _apply_inbound_mcp_derivations(normalized, current) _apply_operations_derivations(normalized, current) + # Derived from the merged scope switches and hour, so it is written only once + # the whole save is known to be valid. + _apply_retention_schedule(normalized, current) return normalized, errors, warnings diff --git a/application/single_app/config.py b/application/single_app/config.py index 98bde7c3c..adc62f128 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.271" +VERSION = "0.261.272" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_retention_policy.py b/application/single_app/functions_retention_policy.py index a9661ec97..159ce405c 100644 --- a/application/single_app/functions_retention_policy.py +++ b/application/single_app/functions_retention_policy.py @@ -6,13 +6,14 @@ This module handles automated deletion of aged conversations and documents based on configurable retention policies for personal, group, and public workspaces. -Version: 0.261.038 +Version: 0.261.272 Implemented in: 0.234.067 Updated in: 0.236.012 - Fixed race condition handling for NotFound errors during deletion Updated in: 0.237.004 - Fixed critical bug where conversations with null/undefined last_activity_at were deleted regardless of age Updated in: 0.237.005 - Fixed field name: use last_updated (actual field) instead of last_activity_at (non-existent) Updated in: 0.250.103 - Applied retention by conversation ownership across current, legacy, and collaboration stores Updated in: 0.261.038 - Stop pending Microsoft 365 delivery before deleting its destination +Updated in: 0.261.272 - Chats grounded in a public workspace follow that workspace's conversation policy while it exists """ from config import * @@ -30,6 +31,7 @@ from functions_debug import debug_print from functions_appinsights import log_event from datetime import datetime, timezone, timedelta +import logging GROUP_SINGLE_USER_CHAT_TYPES = {'group', 'group-single-user', 'group_single_user'} @@ -97,6 +99,33 @@ def get_all_public_workspaces(): return [] +def _get_existing_public_workspace_ids(): + """Return the ids of the public workspaces that exist, or None if they cannot be listed. + + None is deliberately distinct from an empty set. A failed or partial listing must never + read as every public workspace having been deleted, which would hand every grounded chat + to its owner's personal policy. + """ + try: + workspace_ids = set() + for row in cosmos_public_workspaces_container.query_items( + query="SELECT c.id FROM c", + enable_cross_partition_query=True, + ): + workspace_id = str((row or {}).get('id') or '').strip() + if workspace_id: + workspace_ids.add(workspace_id) + return frozenset(workspace_ids) + except Exception as e: + log_event( + "[RETENTION_POLICY] Public workspace listing failed", + {"error_type": type(e).__name__}, + level=logging.ERROR, + ) + debug_print(f"[RETENTION_POLICY] Could not list public workspaces: {e}") + return None + + def resolve_retention_value(value, workspace_type, retention_type, settings=None): """ Resolve a retention value, handling 'default' by looking up organization defaults. @@ -173,6 +202,36 @@ def _is_group_single_user_conversation(conversation_item): return chat_type in GROUP_SINGLE_USER_CHAT_TYPES or bool(_get_primary_group_id(conversation_item)) +def _get_primary_public_workspace_id(conversation_item): + """Return the public workspace a single-user conversation is grounded in.""" + for context_item in list((conversation_item or {}).get('context', []) or []): + if not isinstance(context_item, dict): + continue + if context_item.get('type') != 'primary' or context_item.get('scope') != 'public': + continue + return str(context_item.get('id') or '').strip() + return '' + + +def _is_governed_by_public_workspace(conversation_item, existing_public_workspace_ids): + """Whether a public workspace's policy, rather than the owner's, governs a conversation. + + Public workspaces have no conversation store; a chat grounded in one is a personal + conversation whose primary context names the workspace. It follows that workspace's + policy while the workspace exists. A chat with no workspace id, or whose workspace has + been deleted, stays under its owner's personal policy instead of falling outside every + policy. When the existing workspaces are unknown (``None``), every grounded chat is + treated as governed, so a personal run skips it rather than risk deleting it under the + wrong policy. + """ + public_workspace_id = _get_primary_public_workspace_id(conversation_item) + if not public_workspace_id: + return False + if existing_public_workspace_ids is None: + return True + return public_workspace_id in existing_public_workspace_ids + + def _is_converted_conversation_source(conversation_item): return bool(str((conversation_item or {}).get('collaboration_conversation_id') or '').strip()) @@ -204,14 +263,36 @@ def _build_group_scope_query(timestamp_field): """ +def _build_public_scope_query(timestamp_field): + return f""" + SELECT * FROM c + WHERE IS_ARRAY(c.context) + AND EXISTS( + SELECT VALUE context_item + FROM context_item IN c.context + WHERE context_item.type = 'primary' + AND context_item.scope = 'public' + AND context_item.id = @scope_id + ) + AND IS_DEFINED(c.{timestamp_field}) + AND IS_STRING(c.{timestamp_field}) + AND c.{timestamp_field} < @cutoff_date + """ + + def _build_conversation_retention_sources( workspace_type, cutoff_iso, user_id=None, group_id=None, public_workspace_id=None, + existing_public_workspace_ids=None, ): - """Describe the backing stores governed by a workspace retention policy.""" + """Describe the backing stores governed by a workspace retention policy. + + ``existing_public_workspace_ids`` lets a personal run take back chats grounded in a + public workspace that has since been deleted. Without it, grounded chats are skipped. + """ if workspace_type == 'personal': return [ { @@ -232,6 +313,10 @@ def _build_conversation_retention_sources( ], 'matches_scope': lambda item: ( not _is_group_single_user_conversation(item) + and not _is_governed_by_public_workspace( + item, + existing_public_workspace_ids, + ) and not _is_converted_conversation_source(item) ), }, @@ -316,24 +401,25 @@ def _build_conversation_retention_sources( }, ] + # Public workspaces have no conversation store of their own. A chat grounded in + # one is a personal conversation whose primary context names the workspace, and + # it follows that workspace's policy the way a group-grounded chat follows its + # group's. return [ { - 'name': 'public', - 'container': cosmos_public_conversations_container, - 'messages_container': cosmos_public_messages_container, + 'name': 'public_single_user', + 'container': cosmos_conversations_container, + 'messages_container': cosmos_messages_container, 'timestamp_field': 'last_updated', - 'query': """ - SELECT * FROM c - WHERE c.public_workspace_id = @scope_id - AND IS_DEFINED(c.last_updated) - AND IS_STRING(c.last_updated) - AND c.last_updated < @cutoff_date - """, + 'query': _build_public_scope_query('last_updated'), 'parameters': [ {'name': '@scope_id', 'value': public_workspace_id}, {'name': '@cutoff_date', 'value': cutoff_iso}, ], - 'matches_scope': lambda item: item.get('public_workspace_id') == public_workspace_id, + 'matches_scope': lambda item: ( + _get_primary_public_workspace_id(item) == public_workspace_id + and not _is_converted_conversation_source(item) + ), }, ] @@ -361,6 +447,7 @@ def _delete_standard_conversation_for_retention( return { 'id': conversation_id, 'title': conversation_title, + 'user_id': selected_conversation_item.get('user_id'), source['timestamp_field']: selected_conversation_item.get( source['timestamp_field'] ), @@ -385,6 +472,12 @@ def _delete_standard_conversation_for_retention( conversation_item = live_conversation_item cancel_m365_conversation_deliveries(conversation_id) + # A grounded chat names its workspace in its primary context, not at the top level. + grounded_public_workspace_id = ( + _get_primary_public_workspace_id(conversation_item) + or conversation_item.get('public_workspace_id') + or None + ) if archiving_enabled: archived_item = dict(conversation_item) @@ -401,7 +494,7 @@ def _delete_standard_conversation_for_retention( context=conversation_item.get('context', []), tags=conversation_item.get('tags', []), group_id=_get_primary_group_id(conversation_item) or None, - public_workspace_id=conversation_item.get('public_workspace_id'), + public_workspace_id=grounded_public_workspace_id, additional_context={'deletion_reason': 'retention_policy'}, ) @@ -456,7 +549,7 @@ def _delete_standard_conversation_for_retention( is_archived=archiving_enabled, is_bulk_operation=True, group_id=_get_primary_group_id(conversation_item) or None, - public_workspace_id=conversation_item.get('public_workspace_id'), + public_workspace_id=grounded_public_workspace_id, additional_context={ 'deletion_reason': 'retention_policy', 'retention_source': source['name'], @@ -480,6 +573,8 @@ def _delete_standard_conversation_for_retention( return { 'id': conversation_id, 'title': conversation_title, + # Lets a public workspace's run tell each owner about their own chats. + 'user_id': conversation_item.get('user_id'), source['timestamp_field']: conversation_item.get(source['timestamp_field']), } @@ -594,6 +689,9 @@ def process_personal_retention(): # Pre-load settings once for efficiency settings = get_settings() + + # Chats grounded in a public workspace follow its policy only while it exists. + existing_public_workspace_ids = _get_existing_public_workspace_ids() for user in all_users: user_id = user.get('id') @@ -632,7 +730,8 @@ def process_personal_retention(): conv_results = delete_aged_conversations( user_id=user_id, retention_days=int(conversation_retention_days), - workspace_type='personal' + workspace_type='personal', + existing_public_workspace_ids=existing_public_workspace_ids, ) user_deletion_summary['conversations_deleted'] = conv_results['count'] user_deletion_summary['conversation_details'] = conv_results['details'] @@ -814,11 +913,26 @@ def process_public_retention(): 'document_details': [] } - # Note: Public workspaces do not have a separate conversations container. - # Conversations are only stored in personal (cosmos_conversations_container) or - # group (cosmos_group_conversations_container) workspaces. - # Therefore, we skip conversation processing for public workspaces. - # Only documents are processed for public workspace retention. + # Public workspaces have no conversation store of their own. The chats + # this processes are personal conversations grounded in the workspace, + # which follow its policy the way group-grounded chats follow their group's. + if conversation_retention_days != 'none': + try: + conv_results = delete_aged_conversations( + public_workspace_id=workspace_id, + retention_days=int(conversation_retention_days), + workspace_type='public' + ) + workspace_deletion_summary['conversations_deleted'] = conv_results['count'] + workspace_deletion_summary['conversation_details'] = conv_results['details'] + results['conversations'] += conv_results['count'] + except Exception as e: + log_event( + "[RETENTION_POLICY] Public workspace conversation retention failed", + {"error_type": type(e).__name__, "public_workspace_id": workspace_id}, + level=logging.ERROR, + ) + debug_print(f"Error processing conversations for public workspace {workspace_id}: {e}") # Process documents if document_retention_days != 'none': @@ -837,7 +951,18 @@ def process_public_retention(): # Send notification if anything was deleted if workspace_deletion_summary['conversations_deleted'] > 0 or workspace_deletion_summary['documents_deleted'] > 0: - send_retention_notification(workspace_id, workspace_deletion_summary, 'public') + # The chats belong to individual users, so the workspace's managers + # are told how many were removed and each owner is told which. + send_retention_notification( + workspace_id, + {**workspace_deletion_summary, 'conversation_details': []}, + 'public', + ) + send_public_conversation_owner_notifications( + workspace_id, + workspace_deletion_summary['workspace_name'], + workspace_deletion_summary['conversation_details'], + ) results['workspaces_affected'] += 1 results['details'].append(workspace_deletion_summary) @@ -849,7 +974,14 @@ def process_public_retention(): return results -def delete_aged_conversations(retention_days, workspace_type='personal', user_id=None, group_id=None, public_workspace_id=None): +def delete_aged_conversations( + retention_days, + workspace_type='personal', + user_id=None, + group_id=None, + public_workspace_id=None, + existing_public_workspace_ids=None, +): """ Delete conversations governed by a workspace policy across all backing stores. @@ -859,6 +991,9 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id user_id (str, optional): User ID for personal workspaces group_id (str, optional): Group ID for group workspaces public_workspace_id (str, optional): Public workspace ID for public workspaces + existing_public_workspace_ids (frozenset, optional): For personal runs, the public + workspaces that exist. Chats grounded in any other workspace return to their + owner's policy; when omitted, every grounded chat is skipped. Returns: dict: {'count': int, 'details': list} @@ -874,6 +1009,7 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id user_id=user_id, group_id=group_id, public_workspace_id=public_workspace_id, + existing_public_workspace_ids=existing_public_workspace_ids, ) for source in sources: @@ -949,6 +1085,7 @@ def delete_aged_conversations(retention_days, workspace_type='personal', user_id deleted_details.append({ 'id': conversation_id, 'title': conversation_item.get('title', 'Untitled'), + 'user_id': conversation_item.get('user_id'), source['timestamp_field']: conversation_item.get(source['timestamp_field']), 'already_deleted': True, }) @@ -1183,3 +1320,60 @@ def send_retention_notification(workspace_id, deletion_summary, workspace_type): ) debug_print(f"Sent retention notification to {workspace_type} workspace {workspace_id}") + + +def send_public_conversation_owner_notifications(public_workspace_id, workspace_name, conversation_details): + """Tell each owner which of their chats a public workspace's policy removed. + + A chat grounded in a public workspace is a personal conversation, so its title + belongs to its owner. The workspace's own notification carries only a count; the + titles go here, to each owner, and nowhere else. Records another process had + already deleted are left out, because retention did not remove them. + """ + deleted_by_owner = {} + for detail in conversation_details or []: + if not isinstance(detail, dict) or detail.get('already_deleted'): + continue + owner_id = str(detail.get('user_id') or '').strip() + if owner_id: + deleted_by_owner.setdefault(owner_id, []).append(detail) + + display_name = str(workspace_name or '').strip() or 'a public workspace' + for owner_id, owner_details in deleted_by_owner.items(): + count = len(owner_details) + lines = [ + f"Retention policy for {display_name} automatically deleted {count} " + f"of your conversation{'s' if count != 1 else ''} grounded in it.", + "", + "**Conversations:**", + ] + for detail in owner_details[:10]: + lines.append(f"• {detail.get('title', 'Untitled')}") + if count > 10: + lines.append(f"• ...and {count - 10} more") + + try: + create_notification( + user_id=owner_id, + notification_type='system_announcement', + title='Retention Policy Cleanup', + message="\n".join(lines), + link_url='/chats', + metadata={ + 'conversations_deleted': count, + 'documents_deleted': 0, + 'public_workspace_id': public_workspace_id, + 'deletion_date': datetime.now(timezone.utc).isoformat(), + }, + ) + except Exception as notification_error: + # One owner's notification failing must not stop the others. + log_event( + "[RETENTION_POLICY] Owner notification for a public workspace run failed", + { + "error_type": type(notification_error).__name__, + "public_workspace_id": public_workspace_id, + "user_id": owner_id, + }, + level=logging.WARNING, + ) diff --git a/application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx b/application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx new file mode 100644 index 000000000..a602377c1 --- /dev/null +++ b/application/v2_ui/src/components/admin/ClassificationCategoriesEditor.tsx @@ -0,0 +1,210 @@ +// ClassificationCategoriesEditor.tsx +// The categories users choose from when they classify a document. +// +// The classic pane edits these in a table with an Edit and a Save button per row, and +// keeps the list in a hidden JSON field. Here each row is edited in place and the list +// flows into the page's draft like any other setting, so the save bar saves it with the +// rest of the section. +// +// Order matters: it is the order users see wherever they pick a category, so rows move. +// The preview draws each category with the same badge documents and conversations use. + +import { clsx } from 'clsx'; +import { AlertCircle, ArrowDown, ArrowUp, Plus, Tags, Trash2 } from 'lucide-react'; +import type { AdminField } from '../../lib/adminFields'; +import { + CLASSIFICATION_DEFAULT_COLOR, + CLASSIFICATION_LABEL_MAX_LENGTH, + describeCategoryProblems, + readClassificationCategories, + safeBadgeColor, + type ClassificationCategory, +} from '../../lib/classificationCategories'; +import { ClassificationBadge } from '../documents/documentPresentation'; +import { GlassButton } from '../ui/primitives'; + +const inputClass = clsx( + 'min-h-9 rounded-lg border border-edge bg-surface-1 px-2.5 py-1.5', + 'text-sm text-text-1 placeholder:text-text-3', + 'focus:border-accent focus:outline-none', + 'disabled:cursor-not-allowed disabled:opacity-60', +); + +const iconButtonClass = clsx( + 'rounded-lg p-1.5 text-text-3 transition-colors hover:bg-surface-2 hover:text-text-1', + 'disabled:cursor-not-allowed disabled:opacity-40', +); + +export function ClassificationCategoriesEditor({ + field, + value, + error, + disabled, + onChange, +}: { + field: AdminField; + value: unknown; + error?: string; + disabled?: boolean; + onChange: (next: ClassificationCategory[]) => void; +}) { + const categories = readClassificationCategories(value); + const problems = describeCategoryProblems(categories); + const previewable = categories.filter( + (category, index) => !problems[index] && category.label.trim(), + ); + + const replace = (index: number, patch: Partial) => { + onChange(categories.map((category, i) => (i === index ? { ...category, ...patch } : category))); + }; + + const move = (index: number, delta: number) => { + const target = index + delta; + if (target < 0 || target >= categories.length) { + return; + } + const next = [...categories]; + [next[index], next[target]] = [next[target], next[index]]; + onChange(next); + }; + + return ( +
+
+ {field.label} + + {categories.length} {categories.length === 1 ? 'category' : 'categories'} + +
+ + {field.help ? ( +

{field.help}

+ ) : null} + + {categories.length === 0 ? ( +
+
+ ) : ( + + )} + + onChange([...categories, { label: '', color: CLASSIFICATION_DEFAULT_COLOR }])} + > + + + {previewable.length ? ( +
+

As users will see them

+
+ {previewable.map((category, index) => ( + + ))} +
+
+ ) : null} + + {error ? ( +

+

+ ) : null} +
+ ); +} diff --git a/application/v2_ui/src/components/admin/RetentionOperations.tsx b/application/v2_ui/src/components/admin/RetentionOperations.tsx new file mode 100644 index 000000000..74a19eef4 --- /dev/null +++ b/application/v2_ui/src/components/admin/RetentionOperations.tsx @@ -0,0 +1,709 @@ +// RetentionOperations.tsx +// Run retention now, and reset every chosen period back to the organization defaults. +// +// Both act on everything stored for a workspace type and neither can be undone, so a click +// opens a review rather than acting: the types it may touch, the defaults those types +// follow, and plainly what happens to the data. The review sits inline rather than in a +// dialog so the defaults above stay in view, and so a run that takes minutes on a large +// deployment does not hold the whole page behind a modal. +// +// Both work from saved settings only, because the routes read what is stored. While the +// draft holds an edit they depend on, both are held with a message saying to save first, +// and only workspace types whose retention is on in the saved settings can be chosen. + +import { useEffect, useId, useRef, useState, type ReactNode, type RefObject } from 'react'; +import { clsx } from 'clsx'; +import { + AlertCircle, + AlertTriangle, + CheckCircle2, + CirclePlay, + Loader2, + RotateCcw, + XCircle, + type LucideIcon, +} from 'lucide-react'; +import { api } from '../../lib/apiClient'; +import { asBoolean, type AdminField } from '../../lib/adminFields'; +import { + CONVERSATION_ARCHIVING_KEY, + RETENTION_AFFECTED_NOUNS, + RETENTION_LAST_RUN_KEY, + RETENTION_NEXT_RUN_KEY, + RETENTION_RUN_KEYS, + RETENTION_SCOPES, + RETENTION_SCOPE_LABELS, + RETENTION_SETTING_KEYS, + retentionRequestError, + savedEnabledScopes, + scopeDefaultsSummary, + summarizeRetentionReset, + summarizeRetentionRun, + unsavedKeys, + type RetentionResetSummary, + type RetentionRunSummary, + type RetentionScope, +} from '../../lib/retentionPolicy'; +import type { Json } from '../../lib/types'; +import { GlassButton } from '../ui/primitives'; +import { RETENTION_SCOPE_ICONS } from './retentionScopeIcons'; + +type Phase = 'idle' | 'review' | 'working' | 'done' | 'failed'; + +function plural(count: number, [one, many]: readonly [string, string]): string { + return `${count.toLocaleString()} ${count === 1 ? one : many}`; +} + +/** A row laid out like every other setting: label and help on the left, action on the right. */ +function OperationRow({ field, children }: { field: AdminField; children: ReactNode }) { + return ( +
+
{field.label}
+ {field.help ? ( +

{field.help}

+ ) : null} +
{children}
+
+ ); +} + +function Callout({ tone, children }: { tone: 'warn' | 'danger' | 'ok'; children: ReactNode }) { + const Icon: LucideIcon = tone === 'ok' ? CheckCircle2 : tone === 'danger' ? XCircle : AlertTriangle; + return ( +
+
+ ); +} + +/** Why an action is held, shown under its trigger and inside an open review. */ +function HeldNotice({ children }: { children: ReactNode }) { + return ( +

+

+ ); +} + +function ScopeChoices({ + idPrefix, + legend, + enabled, + selected, + disabled, + describe, + onToggle, +}: { + idPrefix: string; + legend: string; + enabled: readonly RetentionScope[]; + selected: readonly RetentionScope[]; + disabled: boolean; + describe: (scope: RetentionScope) => ReactNode; + onToggle: (scope: RetentionScope, checked: boolean) => void; +}) { + return ( +
+ {legend} + +
+ ); +} + +/** + * The open/close, focus and selection behaviour both reviews share. + * + * Opening moves focus to the review's heading so a keyboard or screen reader user lands in + * it; closing hands focus back to the button that opened it. + */ +function useReview(enabled: readonly RetentionScope[]) { + const [phase, setPhase] = useState('idle'); + const [selected, setSelected] = useState([]); + const triggerRef = useRef(null); + const headingRef = useRef(null); + const returnFocus = useRef(false); + + useEffect(() => { + if (phase === 'review' || phase === 'done' || phase === 'failed') { + headingRef.current?.focus(); + } else if (phase === 'idle' && returnFocus.current) { + returnFocus.current = false; + triggerRef.current?.focus(); + } + }, [phase]); + + return { + phase, + setPhase, + // A type switched off and saved while the review is open drops out of the choice. + chosen: selected.filter((scope) => enabled.includes(scope)), + triggerRef, + headingRef, + open: () => { + setSelected([]); + setPhase('review'); + }, + close: () => { + returnFocus.current = true; + setPhase('idle'); + }, + toggle: (scope: RetentionScope, checked: boolean) => + setSelected((current) => + checked ? [...new Set([...current, scope])] : current.filter((item) => item !== scope), + ), + }; +} + +/** + * The button that opens a review, styled like the other in-row actions (Test connection). + * + * It grows with its label rather than holding a fixed height, so at a large text size on a + * narrow card the label wraps inside the button instead of spilling over the help text. + */ +function ReviewTrigger({ + triggerRef, + icon: Icon, + label, + disabled, + onOpen, +}: { + triggerRef: RefObject; + icon: LucideIcon; + label: string; + disabled: boolean; + onOpen: () => void; +}) { + return ( + + ); +} + +interface OperationProps { + field: AdminField; + /** The saved settings. Both actions work from these alone. */ + settings: Json; + draft: Json; + /** True while the page is saving. */ + disabled?: boolean; +} + +export function RetentionRunNow({ + field, + settings, + draft, + disabled = false, + onStoredSettingsChange, + onOpenSection, +}: OperationProps & { + /** Merge the run's new last and next run into the page's copy of the saved settings. */ + onStoredSettingsChange: (partial: Json) => void; + onOpenSection: (sectionId: string) => void; +}) { + const enabled = savedEnabledScopes(settings); + const held = unsavedKeys(draft, RETENTION_RUN_KEYS).length > 0; + const archiving = asBoolean(settings[CONVERSATION_ARCHIVING_KEY]); + const review = useReview(enabled); + const [summary, setSummary] = useState(null); + const [failure, setFailure] = useState(''); + const idPrefix = useId(); + const headingId = `${idPrefix}-heading`; + + const refreshSchedule = async () => { + try { + const response = await api.get<{ settings?: Json }>('/api/admin/retention-policy/settings'); + const stored = response.settings ?? {}; + onStoredSettingsChange({ + [RETENTION_LAST_RUN_KEY]: stored[RETENTION_LAST_RUN_KEY] ?? null, + [RETENTION_NEXT_RUN_KEY]: stored[RETENTION_NEXT_RUN_KEY] ?? null, + }); + } catch { + // The run's own outcome is already on screen; a stale schedule readout is not + // worth a second error beside it, and "Check again" can read it later. + } + }; + + const run = async () => { + const scopes = review.chosen; + if (!scopes.length || held) { + return; + } + review.setPhase('working'); + setSummary(null); + setFailure(''); + try { + const response = await api.post('/api/admin/retention-policy/execute', { scopes }); + setSummary(summarizeRetentionRun(response, scopes)); + review.setPhase('done'); + } catch (error) { + setFailure(retentionRequestError(error, 'run')); + review.setPhase('failed'); + } finally { + void refreshSchedule(); + } + }; + + if (review.phase === 'idle') { + return ( + + + {held ? ( + + Save or discard your changes first. Retention runs on saved settings only. + + ) : null} + + ); + } + + const working = review.phase === 'working'; + + return ( + +
+ {review.phase === 'done' && summary ? ( + <> +

+ {summary.ok && !summary.errorCount ? 'Retention finished' : 'Retention finished with problems'} +

+
+ +

+ {summary.conversations || summary.documents + ? `Removed ${plural(summary.conversations, ['conversation', 'conversations'])} and ${plural(summary.documents, ['document', 'documents'])}.` + : 'Nothing was past its period, so nothing was removed.'} +

+ {summary.errorCount ? ( +

+ The run reported {plural(summary.errorCount, ['problem', 'problems'])}. Check + the application logs for the cause before running it again. +

+ ) : null} + + {summary.rows.length ? ( +
+ + + + + + + + + + + {summary.rows.map((row) => ( + + + + + + + ))} + +
Workspace typeConversationsDocumentsAffected
+ {RETENTION_SCOPE_LABELS[row.scope]} + {row.conversations.toLocaleString()}{row.documents.toLocaleString()} + {plural(row.affected, RETENTION_AFFECTED_NOUNS[row.scope])} +
+
+ ) : null} +
+
+ + Done + +
+ + ) : review.phase === 'failed' ? ( + <> +

+ Retention did not report back +

+
+ +

{failure}

+ +
+
+ + Close + +
+ + ) : ( + <> +

+ Review before running +

+

+ Every user, group or workspace keeps the period it chose. The rest follow + the defaults shown. +

+ scopeDefaultsSummary(settings, scope, 'run')} + onToggle={review.toggle} + /> +
+ +

+ + This deletes data now and cannot be undone. + {' '} + Conversations idle and documents unchanged for longer than their + period are removed from the chosen types. +

+

Documents are deleted permanently, with their search entries and stored files.

+

+ {archiving + ? 'Conversations are copied to the archive first, because Conversation Archiving is on. ' + : 'Conversations are deleted permanently, because Conversation Archiving is off. '} + +

+

Affected owners are notified of what was removed.

+ +
+ {held ? ( + + Save or discard your changes first. Retention runs on saved settings only. + + ) : null} + {working ? ( +

+ Running. On a large deployment this can take several minutes; the result + appears here when it finishes. +

+ ) : null} +
+ + Cancel + + void run()} + > + {working ? ( + +
+ + )} +
+ + ); +} + +export function RetentionResetDefaults({ field, settings, draft, disabled = false }: OperationProps) { + const enabled = savedEnabledScopes(settings); + const held = unsavedKeys(draft, RETENTION_SETTING_KEYS).length > 0; + const review = useReview(enabled); + const [summary, setSummary] = useState(null); + const [failure, setFailure] = useState(''); + const idPrefix = useId(); + const headingId = `${idPrefix}-heading`; + + const reset = async () => { + const scopes = review.chosen; + if (!scopes.length || held) { + return; + } + review.setPhase('working'); + setSummary(null); + setFailure(''); + try { + const response = await api.post('/api/admin/retention-policy/force-push', { scopes }); + setSummary(summarizeRetentionReset(response, scopes)); + review.setPhase('done'); + } catch (error) { + setFailure(retentionRequestError(error, 'reset')); + review.setPhase('failed'); + } + }; + + if (review.phase === 'idle') { + return ( + + + {held ? ( + + Save or discard your changes first. The reset applies the saved defaults only. + + ) : null} + + ); + } + + const working = review.phase === 'working'; + + return ( + +
+ {review.phase === 'done' && summary ? ( + <> +

+ {summary.ok ? 'Defaults applied' : 'The reset did not finish'} +

+
+ +

+ {summary.ok + ? `Reset ${plural(summary.total, ['retention policy', 'retention policies'])} to follow the organization defaults.` + : 'Some policies may already follow the defaults. Check the application logs, then try again.'} +

+ + {summary.rows.length ? ( +
    + {summary.rows.map((row) => ( +
  • + {RETENTION_SCOPE_LABELS[row.scope]} + + {plural(row.updated, RETENTION_AFFECTED_NOUNS[row.scope])} + +
    • + ))} +
+ ) : null} +
+
+ + Done + +
+ + ) : review.phase === 'failed' ? ( + <> +

+ The reset did not report back +

+
+ +

{failure}

+ +
+
+ + Close + +
+ + ) : ( + <> +

+ Review before resetting +

+

+ Everyone in the chosen types goes back to following these defaults. +

+ scopeDefaultsSummary(settings, scope, 'reset')} + onToggle={review.toggle} + /> +
+ +

+ This cannot be undone.{' '} + Every user, group and public workspace of the chosen types loses the + period it chose. They would have to choose again. +

+

Nothing is deleted by the reset itself; the next run applies the defaults.

+ +
+ {held ? ( + + Save or discard your changes first. The reset applies the saved defaults only. + + ) : null} + {working ? ( +

+ Resetting. This visits every user, group or workspace of the chosen types. +

+ ) : null} +
+ + Cancel + + void reset()} + > + {working ? ( + +
+ + )} +
+ + ); +} diff --git a/application/v2_ui/src/components/admin/RetentionSchedule.tsx b/application/v2_ui/src/components/admin/RetentionSchedule.tsx new file mode 100644 index 000000000..1e8b512e9 --- /dev/null +++ b/application/v2_ui/src/components/admin/RetentionSchedule.tsx @@ -0,0 +1,191 @@ +// RetentionSchedule.tsx +// The daily retention run: when it happens, when it last happened, and when it next will. +// +// The classic pane shows the hour as a select and the last and next run as raw ISO +// strings. Here the hour reads as UTC with the administrator's own clock beside it, and +// both timestamps read as a date plus how far away they are, because "next run in 3 +// hours" is what someone checking the schedule actually wants to know. +// +// The hour is stored as an int -- execute_retention_policy hands it to +// datetime.replace(hour=...) -- which is why this is a component rather than a generic +// select, whose options are strings. + +import { useState, type ReactNode } from 'react'; +import { clsx } from 'clsx'; +import { AlertCircle, Loader2, RefreshCw } from 'lucide-react'; +import { api } from '../../lib/apiClient'; +import type { AdminField } from '../../lib/adminFields'; +import { + RETENTION_LAST_RUN_KEY, + RETENTION_NEXT_RUN_KEY, + describeRunTime, + localTimeForUtcHour, + projectNextRun, + readExecutionHour, + utcHourLabel, + willReschedule, +} from '../../lib/retentionPolicy'; +import type { Json } from '../../lib/types'; +import { FieldShell, inputClass } from './fields'; + +const HOURS = Array.from({ length: 24 }, (_, hour) => hour); + +interface ScheduleResponse { + settings?: Json; +} + +export function RetentionSchedule({ + field, + value, + settings, + draft, + error, + disabled, + onChange, + onStoredSettingsChange, +}: { + field: AdminField; + value: unknown; + /** The saved settings, which hold the last and next run. */ + settings: Json; + draft: Json; + error?: string; + disabled?: boolean; + onChange: (hour: number) => void; + /** Merge freshly read schedule values into the page's copy of the saved settings. */ + onStoredSettingsChange: (partial: Json) => void; +}) { + const [checking, setChecking] = useState(false); + const [checkFailed, setCheckFailed] = useState(false); + + const id = `admin-field-${field.key ?? 'retention-schedule'}`; + const hour = readExecutionHour(value); + const now = new Date(); + const local = localTimeForUtcHour(hour, now); + const last = describeRunTime(settings[RETENTION_LAST_RUN_KEY], now); + const next = describeRunTime(settings[RETENTION_NEXT_RUN_KEY], now); + const projected = willReschedule(settings, draft) + ? describeRunTime(projectNextRun(hour, now), now) + : null; + + const check = async () => { + setChecking(true); + setCheckFailed(false); + try { + const response = await api.get('/api/admin/retention-policy/settings'); + const stored = response.settings ?? {}; + onStoredSettingsChange({ + [RETENTION_LAST_RUN_KEY]: stored[RETENTION_LAST_RUN_KEY] ?? null, + [RETENTION_NEXT_RUN_KEY]: stored[RETENTION_NEXT_RUN_KEY] ?? null, + }); + } catch { + setCheckFailed(true); + } finally { + setChecking(false); + } + }; + + let nextRun: ReactNode; + if (projected) { + nextRun = ( + <> + + · {projected.relative}, once you save + + ); + } else if (next && next.past) { + nextRun = ( + <> + + · due now; the scheduler checks every 5 minutes + + ); + } else if (next) { + nextRun = ( + <> + + · {next.relative} + + ); + } else { + nextRun = ( + + Not set yet. It is worked out when a run time is saved or the next run finishes. + + ); + } + + return ( + +
+ + {local ? ( +

That is {local} on your clock.

+ ) : null} +
+ +
+ {/* Each pair wraps its value below the term once the card is too narrow to + hold both on one line, rather than squeezing the value into a sliver. */} +
+
Last run
+
+ {last ? ( + <> + + · {last.relative} + + ) : ( + Never run + )} +
+
+
+
Next run
+
{nextRun}
+
+
+ +
+ + {checkFailed ? ( + + + ) : null} +
+ + ); +} diff --git a/application/v2_ui/src/components/admin/agentSectionAppearance.ts b/application/v2_ui/src/components/admin/agentSectionAppearance.ts index 3e19d5e6e..fd3b79df6 100644 --- a/application/v2_ui/src/components/admin/agentSectionAppearance.ts +++ b/application/v2_ui/src/components/admin/agentSectionAppearance.ts @@ -2,13 +2,15 @@ import { UserRound, UsersRound } from 'lucide-react'; import type { SettingsSectionAppearance } from './SettingsSection'; +import { RETENTION_SCOPE_ICONS } from './retentionScopeIcons'; // Presentation only: field order, visibility, and status still come from the schema. // // Section icons now come from the navigation definition, and which switch leads which // settings is derived from `depends_on`, so the Agent Runtime emphasis no longer needs -// declaring here. What remains is the one cue the schema cannot express: whether a -// workspace permission applies to a person or to a group. +// declaring here. What remains are cues the schema cannot express: whether a workspace +// permission applies to a person or to a group, and which workspace type each retention +// switch governs. export const agentSectionAppearances: Readonly< Partial> > = { @@ -21,4 +23,13 @@ export const agentSectionAppearances: Readonly< allow_group_custom_endpoints: { Icon: UsersRound }, }, }, + // The three workspace types are peers, so the first is not promoted to the switch the + // card is about, which is what the derived hierarchy would otherwise do with it. + 'retention-policy-section': { + fields: { + enable_retention_policy_personal: { Icon: RETENTION_SCOPE_ICONS.personal, emphasis: 'none' }, + enable_retention_policy_group: { Icon: RETENTION_SCOPE_ICONS.group }, + enable_retention_policy_public: { Icon: RETENTION_SCOPE_ICONS.public }, + }, + }, }; diff --git a/application/v2_ui/src/components/admin/retentionScopeIcons.ts b/application/v2_ui/src/components/admin/retentionScopeIcons.ts new file mode 100644 index 000000000..9560a0bf8 --- /dev/null +++ b/application/v2_ui/src/components/admin/retentionScopeIcons.ts @@ -0,0 +1,12 @@ +// retentionScopeIcons.ts +// One icon per workspace type, shared by the Retention Policy switches and the run and +// reset reviews so a type looks the same wherever it is named. + +import { Globe, User, Users, type LucideIcon } from 'lucide-react'; +import type { RetentionScope } from '../../lib/retentionPolicy'; + +export const RETENTION_SCOPE_ICONS: Readonly> = { + personal: User, + group: Users, + public: Globe, +}; diff --git a/application/v2_ui/src/lib/classificationCategories.ts b/application/v2_ui/src/lib/classificationCategories.ts new file mode 100644 index 000000000..e3109cc6e --- /dev/null +++ b/application/v2_ui/src/lib/classificationCategories.ts @@ -0,0 +1,75 @@ +// classificationCategories.ts +// Reading and checking the document classification categories edited in Admin Settings. +// +// The server is authoritative (admin_settings_fields._normalize_document_classification_categories) +// and refuses a bad list on save. These checks run while typing so each problem is shown +// on the row that has it, before Save is ever pressed. + +export interface ClassificationCategory { + label: string; + color: string; +} + +/** Mirrors DOCUMENT_CLASSIFICATION_LABEL_MAX_LENGTH in admin_settings_fields.py. */ +export const CLASSIFICATION_LABEL_MAX_LENGTH = 80; + +/** The colour a new category starts with, matching the classic editor. */ +export const CLASSIFICATION_DEFAULT_COLOR = '#808080'; + +const HEX_COLOR = /^#[0-9a-fA-F]{6}$/; + +export function isHexColor(value: string): boolean { + return HEX_COLOR.test(value.trim()); +} + +/** + * A colour safe to paint with, or undefined. + * + * The stored list is administrator data from the database and may predate validation, so + * only a six-digit hex ever reaches a style; anything else falls back to the neutral badge. + */ +export function safeBadgeColor(value: string): string | undefined { + return isHexColor(value) ? value.trim() : undefined; +} + +/** Read the stored value defensively. */ +export function readClassificationCategories(value: unknown): ClassificationCategory[] { + if (!Array.isArray(value)) { + return []; + } + return value + .filter((item): item is Record => Boolean(item) && typeof item === 'object') + .map((item) => ({ + label: typeof item.label === 'string' ? item.label : '', + color: typeof item.color === 'string' ? item.color : '', + })); +} + +/** + * The problem with each row, or null when it would save. + * + * A repeated label is reported on the later row, naming the earlier one, because documents + * store the label itself: two categories with one label could never be told apart. + */ +export function describeCategoryProblems(categories: ClassificationCategory[]): (string | null)[] { + const firstUse = new Map(); + return categories.map((category, index) => { + const label = category.label.trim(); + if (!label) { + return 'Give this category a label.'; + } + if (label.length > CLASSIFICATION_LABEL_MAX_LENGTH) { + return `Keep the label to ${CLASSIFICATION_LABEL_MAX_LENGTH} characters.`; + } + const folded = label.toLocaleLowerCase(); + const earlier = firstUse.get(folded); + if (earlier !== undefined) { + return `Same label as category ${earlier + 1}. Each label must be unique.`; + } + firstUse.set(folded, index); + if (!isHexColor(category.color)) { + return 'Use a six-digit hex colour, such as #808080.'; + } + return null; + }); +} diff --git a/application/v2_ui/src/lib/retentionPolicy.ts b/application/v2_ui/src/lib/retentionPolicy.ts new file mode 100644 index 000000000..62c8c7151 --- /dev/null +++ b/application/v2_ui/src/lib/retentionPolicy.ts @@ -0,0 +1,406 @@ +// retentionPolicy.ts +// Pure helpers behind the Retention Policy controls in Admin Settings. +// +// Kept apart from the components so the decisions that have to be exactly right -- which +// workspace types a run may touch, whether unsaved edits block it, what an hour or a +// stored timestamp reads as, and how the server's response is summarised -- can be +// executed in a test instead of judged from a screenshot. +// +// Nothing here talks to the network. The components call the existing +// /api/admin/retention-policy/* routes and hand the responses to these functions. + +import { ApiError } from './apiClient'; +import { asBoolean } from './adminFields'; +import type { Json } from './types'; + +export type RetentionScope = 'personal' | 'group' | 'public'; + +export const RETENTION_SCOPES: readonly RetentionScope[] = ['personal', 'group', 'public']; + +export const RETENTION_SCOPE_LABELS: Readonly> = { + personal: 'Personal workspaces', + group: 'Group workspaces', + public: 'Public workspaces', +}; + +/** What a run's "affected" count is counted in, per type. */ +export const RETENTION_AFFECTED_NOUNS: Readonly> = { + personal: ['user', 'users'], + group: ['group', 'groups'], + public: ['workspace', 'workspaces'], +}; + +export const RETENTION_EXECUTION_HOUR_KEY = 'retention_policy_execution_hour'; +export const RETENTION_LAST_RUN_KEY = 'retention_policy_last_run'; +export const RETENTION_NEXT_RUN_KEY = 'retention_policy_next_run'; +export const CONVERSATION_ARCHIVING_KEY = 'enable_conversation_archiving'; + +/** Mirrors RETENTION_EXECUTION_HOUR_DEFAULT in admin_settings_fields.py. */ +export const RETENTION_EXECUTION_HOUR_DEFAULT = 2; + +export interface RetentionScopeKeys { + enabled: string; + conversation: string; + document: string; +} + +export function retentionScopeKeys(scope: RetentionScope): RetentionScopeKeys { + return { + enabled: `enable_retention_policy_${scope}`, + conversation: `default_retention_conversation_${scope}`, + document: `default_retention_document_${scope}`, + }; +} + +/** + * Every stored setting a reset depends on. + * + * An unsaved edit to any of these blocks Reset: the server works from what is saved, so + * running it beside a different, unsaved set of defaults would apply something other than + * what is on screen. + */ +export const RETENTION_SETTING_KEYS: readonly string[] = [ + ...RETENTION_SCOPES.flatMap((scope) => { + const keys = retentionScopeKeys(scope); + return [keys.enabled, keys.conversation, keys.document]; + }), + RETENTION_EXECUTION_HOUR_KEY, +]; + +/** A run also reads whether to archive conversations before removing them. */ +export const RETENTION_RUN_KEYS: readonly string[] = [ + ...RETENTION_SETTING_KEYS, + CONVERSATION_ARCHIVING_KEY, +]; + +/** Which of `keys` hold an unsaved edit. */ +export function unsavedKeys(draft: Json, keys: readonly string[]): string[] { + return keys.filter((key) => Object.prototype.hasOwnProperty.call(draft, key)); +} + +/** Workspace types whose retention is switched on in the saved settings. */ +export function savedEnabledScopes(settings: Json): RetentionScope[] { + return RETENTION_SCOPES.filter((scope) => asBoolean(settings[retentionScopeKeys(scope).enabled])); +} + +/** + * A retention period as people read it. + * + * Mirrors resolve_retention_value on the server: anything that is not a positive whole + * number of days means nothing is deleted. + */ +export function retentionPeriodLabel(value: unknown): string { + const days = readRetentionDays(value); + return days ? `${days} day${days === 1 ? '' : 's'}` : 'No automatic deletion'; +} + +/** "deleted after 30 days" or "kept", for a sentence about what a default does. */ +export function retentionDefaultPhrase(value: unknown): string { + const days = readRetentionDays(value); + return days ? `deleted after ${days} day${days === 1 ? '' : 's'}` : 'kept'; +} + +/** + * What a workspace type's saved defaults mean for a run or a reset, in one sentence. + * + * When both defaults keep everything the useful fact is who a run can still touch -- only + * those who chose their own period -- so that is what is said instead of "kept, kept". + */ +export function scopeDefaultsSummary( + settings: Json, + scope: RetentionScope, + action: 'run' | 'reset', +): string { + const keys = retentionScopeKeys(scope); + const conversation = retentionDefaultPhrase(settings[keys.conversation]); + const document = retentionDefaultPhrase(settings[keys.document]); + if (conversation === 'kept' && document === 'kept') { + return action === 'run' + ? `Defaults keep everything, so only ${RETENTION_AFFECTED_NOUNS[scope][1]} with their own period are affected.` + : 'Will follow defaults that keep everything.'; + } + const detail = `conversations ${conversation}, documents ${document}.`; + return action === 'run' ? `Defaults: ${detail}` : `Will follow: ${detail}`; +} + +function readRetentionDays(value: unknown): number | null { + const text = String(value ?? '').trim(); + if (!/^\d+$/.test(text)) { + return null; + } + const days = Number.parseInt(text, 10); + return days > 0 ? days : null; +} + +/** The saved hour, read defensively: settings written by hand may hold a string. */ +export function readExecutionHour(value: unknown): number { + const parsed = typeof value === 'number' ? value : Number.parseInt(String(value ?? ''), 10); + return Number.isInteger(parsed) && parsed >= 0 && parsed <= 23 + ? parsed + : RETENTION_EXECUTION_HOUR_DEFAULT; +} + +/** The keys whose saved change moves the next run. Mirrors RETENTION_SCHEDULE_KEYS. */ +export const RETENTION_SCHEDULE_KEYS: readonly string[] = [ + ...RETENTION_SCOPES.map((scope) => retentionScopeKeys(scope).enabled), + RETENTION_EXECUTION_HOUR_KEY, +]; + +/** + * Whether saving the draft will move the next run. + * + * Mirrors _apply_retention_schedule: a schedule key has to be in the save, and either its + * value differs from what is stored or no next run is stored yet. A switch flipped and + * flipped back is still in the draft but moves nothing. + */ +export function willReschedule(settings: Json, draft: Json): boolean { + const submitted = unsavedKeys(draft, RETENTION_SCHEDULE_KEYS); + if (!submitted.length) { + return false; + } + const changed = submitted.some((key) => draft[key] !== settings[key]); + return changed || !settings[RETENTION_NEXT_RUN_KEY]; +} + +/** + * The next run a save would schedule for `hour`, as an ISO timestamp. + * + * Mirrors compute_retention_next_run: today at that hour UTC, or tomorrow once it has + * passed. + */ +export function projectNextRun(hour: number, now: Date = new Date()): string { + const next = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, 0, 0, 0), + ); + if (next.getTime() <= now.getTime()) { + next.setUTCDate(next.getUTCDate() + 1); + } + return next.toISOString(); +} + +/** `02:00 UTC (2 AM)` -- the option wording for the run-time select. */ +export function utcHourLabel(hour: number): string { + const clock = `${String(hour).padStart(2, '0')}:00 UTC`; + const spoken = + hour === 0 ? 'midnight' : hour === 12 ? 'noon' : hour < 12 ? `${hour} AM` : `${hour - 12} PM`; + return `${clock} (${spoken})`; +} + +/** + * The run hour on the reader's own clock, or null when their clock already reads UTC. + * + * Worked out for today's date, so a daylight-saving change is reflected the day it happens. + * `timeZone` and `locale` exist for tests; the browser's own are used otherwise. + */ +export function localTimeForUtcHour( + hour: number, + now: Date = new Date(), + { locale, timeZone }: { locale?: string; timeZone?: string } = {}, +): string | null { + const instant = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, 0, 0), + ); + const parts = new Intl.DateTimeFormat('en-US', { + hour: 'numeric', + minute: 'numeric', + hourCycle: 'h23', + timeZone, + }).formatToParts(instant); + const localHour = Number(parts.find((part) => part.type === 'hour')?.value); + const localMinute = Number(parts.find((part) => part.type === 'minute')?.value); + if (localHour === hour && localMinute === 0) { + return null; + } + return new Intl.DateTimeFormat(locale, { + hour: 'numeric', + minute: '2-digit', + timeZoneName: 'short', + timeZone, + }).format(instant); +} + +/** Parse a stored ISO timestamp, tolerating Python's six fractional digits. */ +export function parseStoredTimestamp(value: unknown): Date | null { + const text = typeof value === 'string' ? value.trim() : ''; + if (!text) { + return null; + } + const parsed = new Date(text.replace(/(\.\d{3})\d+/, '$1')); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +/** "in 12 hours", "yesterday", "3 days ago". */ +export function relativeTimeLabel(date: Date, now: Date = new Date(), locale?: string): string { + const difference = date.getTime() - now.getTime(); + const distance = Math.abs(difference); + const minute = 60_000; + const hour = 60 * minute; + const day = 24 * hour; + if (distance < minute) { + return difference >= 0 ? 'in under a minute' : 'just now'; + } + const [amount, unit]: [number, Intl.RelativeTimeFormatUnit] = + distance < hour + ? [Math.round(distance / minute), 'minute'] + : distance < day + ? [Math.round(distance / hour), 'hour'] + : [Math.round(distance / day), 'day']; + return new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }).format( + difference >= 0 ? amount : -amount, + unit, + ); +} + +export interface RunTimeReadout { + /** The moment in UTC, which is what the schedule is set in. */ + absolute: string; + relative: string; + /** Whether the moment has already passed. */ + past: boolean; + iso: string; +} + +export function describeRunTime( + value: unknown, + now: Date = new Date(), + locale?: string, +): RunTimeReadout | null { + const date = parseStoredTimestamp(value); + if (!date) { + return null; + } + return { + absolute: new Intl.DateTimeFormat(locale, { + weekday: 'short', + month: 'short', + day: 'numeric', + year: 'numeric', + hour: 'numeric', + minute: '2-digit', + timeZone: 'UTC', + timeZoneName: 'short', + }).format(date), + relative: relativeTimeLabel(date, now, locale), + past: date.getTime() <= now.getTime(), + iso: date.toISOString(), + }; +} + +function readCount(value: unknown): number { + const parsed = typeof value === 'number' ? value : Number(value); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 0; +} + +function asRecord(value: unknown): Record { + return value && typeof value === 'object' ? (value as Record) : {}; +} + +function readScopes(value: unknown, fallback: readonly RetentionScope[]): RetentionScope[] { + if (!Array.isArray(value)) { + return [...fallback]; + } + const named = value.filter((item): item is RetentionScope => + (RETENTION_SCOPES as readonly unknown[]).includes(item), + ); + return RETENTION_SCOPES.filter((scope) => named.includes(scope)); +} + +export interface RetentionRunRow { + scope: RetentionScope; + conversations: number; + documents: number; + affected: number; +} + +export interface RetentionRunSummary { + ok: boolean; + rows: RetentionRunRow[]; + conversations: number; + documents: number; + /** + * How many problems the run reported. The server's text is not shown: it can be a raw + * exception message, which is for the logs rather than the browser. + */ + errorCount: number; +} + +/** Summarise `POST /api/admin/retention-policy/execute`. */ +export function summarizeRetentionRun( + response: unknown, + requested: readonly RetentionScope[], +): RetentionRunSummary { + const body = asRecord(response); + const results = asRecord(body.results); + const scopes = readScopes(results.scopes_processed, requested); + const rows = scopes.map((scope) => { + const counts = asRecord(results[scope]); + return { + scope, + conversations: readCount(counts.conversations), + documents: readCount(counts.documents), + affected: readCount(scope === 'personal' ? counts.users_affected : counts.workspaces_affected), + }; + }); + const errors = Array.isArray(results.errors) ? results.errors.length : 0; + return { + ok: body.success !== false && results.success !== false, + rows, + conversations: rows.reduce((total, row) => total + row.conversations, 0), + documents: rows.reduce((total, row) => total + row.documents, 0), + errorCount: errors, + }; +} + +export interface RetentionResetSummary { + ok: boolean; + rows: { scope: RetentionScope; updated: number }[]; + total: number; +} + +/** Summarise `POST /api/admin/retention-policy/force-push`. */ +export function summarizeRetentionReset( + response: unknown, + requested: readonly RetentionScope[], +): RetentionResetSummary { + const body = asRecord(response); + const details = asRecord(body.details); + const rows = readScopes(body.scopes, requested).map((scope) => ({ + scope, + updated: readCount(details[scope]), + })); + return { + ok: body.success !== false, + rows, + total: readCount(body.updated_count) || rows.reduce((total, row) => total + row.updated, 0), + }; +} + +/** + * What to say when a run or reset request fails. + * + * A 400 carries a validation message the route wrote for people, so it is shown. Anything + * else is described rather than quoted, because a 500 from these routes can carry raw + * exception text. A gateway timeout or a dropped connection gets its own wording: the work + * keeps going on the server after the browser stops waiting. + */ +export function retentionRequestError(error: unknown, action: 'run' | 'reset'): string { + if (error instanceof ApiError) { + if (error.status === 400) { + const payload = asRecord(error.payload); + if (typeof payload.error === 'string' && payload.error.trim()) { + return payload.error.trim(); + } + } + if (error.isAuthError) { + return 'Your session no longer has administrator access. Sign in again, then retry.'; + } + if (error.status !== 502 && error.status !== 503 && error.status !== 504) { + return action === 'run' + ? 'Retention could not finish. Check the application logs for the cause.' + : 'The reset could not finish. Check the application logs for the cause.'; + } + } + return action === 'run' + ? 'The page stopped waiting before retention reported back. It may still be running; check Last run in a few minutes.' + : 'The page stopped waiting before the reset reported back. Some workspaces may already follow the defaults.'; +} diff --git a/application/v2_ui/src/pages/AdminSettingsPage.tsx b/application/v2_ui/src/pages/AdminSettingsPage.tsx index 380c88064..47459860f 100644 --- a/application/v2_ui/src/pages/AdminSettingsPage.tsx +++ b/application/v2_ui/src/pages/AdminSettingsPage.tsx @@ -41,6 +41,7 @@ import { BrandingImageField } from '../components/admin/BrandingImageField'; import { ChatDefaultModel } from '../components/admin/ChatDefaultModel'; import { CapabilityModelPicker } from '../components/admin/CapabilityModelPicker'; import { ChatModeNotice } from '../components/admin/ChatModeNotice'; +import { ClassificationCategoriesEditor } from '../components/admin/ClassificationCategoriesEditor'; import { ConnectionTest } from '../components/admin/ConnectionTest'; import { ControlCenterAccessMatrix } from '../components/admin/ControlCenterAccessMatrix'; import { CustomPagesTable } from '../components/admin/CustomPagesTable'; @@ -72,6 +73,8 @@ import { GlobalAgentsManager } from '../components/admin/GlobalAgentsManager'; import { RefreshScheduleStatus } from '../components/admin/RefreshScheduleStatus'; import { RelatedSectionLink } from '../components/admin/RelatedSectionLink'; import { RestartStatus } from '../components/admin/RestartStatus'; +import { RetentionResetDefaults, RetentionRunNow } from '../components/admin/RetentionOperations'; +import { RetentionSchedule } from '../components/admin/RetentionSchedule'; import { SectionGuide, hasSectionGuide } from '../components/admin/guides/sectionGuides'; import { SaveBar } from '../components/admin/SaveBar'; import { SecretField } from '../components/admin/SecretField'; @@ -780,6 +783,18 @@ export function AdminSettingsPage({ focusSection }: { focusSection?: string } = const readSibling = (key: string, fallback = '') => asString(readFieldValue(READ_ONLY_REF(key), settings, draft), fallback); + /** + * Fold values the server changed outside a save into the page's copy of the settings. + * + * A retention run writes its own last and next run. Without this the schedule beside + * it would keep showing the times from when the page was opened. + */ + const mergeStoredSettings = useCallback((partial: Json) => { + setData((current) => + current ? { ...current, settings: { ...current.settings, ...partial } } : current, + ); + }, []); + /** * Show a category from its first section. * @@ -1249,6 +1264,53 @@ export function AdminSettingsPage({ focusSection }: { focusSection?: string } = help={field.help} /> ); + case 'retention-schedule': + return ( + field.key && setValue(field.key, next)} + onStoredSettingsChange={mergeStoredSettings} + /> + ); + case 'retention-reset-defaults': + return ( + + ); + case 'retention-run-now': + return ( + + ); + case 'document-classification-categories': + return ( + field.key && setValue(field.key, next)} + /> + ); case 'control-center-refresh-schedule': return (