diff --git a/application/single_app/config.py b/application/single_app/config.py index c9ee098da..9340aec68 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.212" +VERSION = "0.250.213" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_group_workflows.py b/application/single_app/functions_group_workflows.py index 64add68b8..1563c9b34 100644 --- a/application/single_app/functions_group_workflows.py +++ b/application/single_app/functions_group_workflows.py @@ -35,7 +35,6 @@ WORKFLOW_TASK_RUNNER_TYPES, WORKFLOW_TRIGGER_TYPES, _build_default_model_summary, - _normalize_alert_priority, _normalize_bool, _normalize_document_action_config, _normalize_schedule, @@ -48,6 +47,7 @@ get_workflow_max_tasks, ) from functions_settings import get_settings, normalize_model_endpoints +from functions_workflow_alerts import normalize_workflow_alert_settings GROUP_WORKFLOW_MEMBER_ROLES = ("Owner", "Admin", "DocumentManager", "User") @@ -479,9 +479,12 @@ def save_group_workflow(group_id, workflow_data, actor_user_id, user_info=None): ), default=False, ) if url_access_enabled else False - alert_priority = _normalize_alert_priority( - workflow_data.get('alert_priority', (existing_workflow or {}).get('alert_priority', 'none')) + alert_settings = normalize_workflow_alert_settings( + workflow_data, + existing_workflow=existing_workflow, + task_ids=[task.get('id') for task in tasks], ) + alert_priority = alert_settings['alert_priority'] error_handling = _normalize_workflow_error_handling(workflow_data, existing_workflow=existing_workflow) default_chat_capabilities_enabled = ( (existing_workflow or {}).get('chat_capabilities_enabled', False) @@ -566,6 +569,9 @@ def save_group_workflow(group_id, workflow_data, actor_user_id, user_info=None): 'URL Access authorized at', ) if url_access_authorized else '', 'alert_priority': alert_priority, + 'alert_mode': alert_settings['alert_mode'], + 'alert_rules': alert_settings['alert_rules'], + 'alert_evaluation': alert_settings['alert_evaluation'], 'schedule': schedule, 'document_action': document_action, 'analyze': analyze, diff --git a/application/single_app/functions_notifications.py b/application/single_app/functions_notifications.py index 97b897283..d8865fd43 100644 --- a/application/single_app/functions_notifications.py +++ b/application/single_app/functions_notifications.py @@ -29,6 +29,10 @@ WORKFLOW_ALERT_NOTIFICATION_TYPE = 'workflow_priority_alert' KEY_VAULT_SECRET_REMINDER_NOTIFICATION_TYPE = 'key_vault_secret_expiring' WORKFLOW_ALERT_PRIORITY_CONFIG = { + 'info': { + 'icon': 'bi-info-circle', + 'color': 'info', + }, 'low': { 'icon': 'bi-bell', 'color': 'info', @@ -41,7 +45,16 @@ 'icon': 'bi-exclamation-triangle', 'color': 'danger', }, + 'critical': { + 'icon': 'bi-exclamation-octagon', + 'color': 'danger', + }, } +# A run that errored is presented differently from a run that found something, +# independently of how loud the owner made the severity. +WORKFLOW_ALERT_FAILURE_CATEGORY = 'failure' +WORKFLOW_ALERT_FAILURE_ICON = 'bi-x-octagon' +WORKFLOW_ALERT_DELIVERY_NOTIFY_ONLY = 'notify_only' # Notification type registry for extensibility NOTIFICATION_TYPES = { @@ -212,13 +225,32 @@ def _get_workflow_alert_priority(notification): return priority +def _get_workflow_alert_category(notification): + metadata = notification.get('metadata') or {} + category = str(metadata.get('category') or 'alert').strip().lower() + return category if category in {'alert', WORKFLOW_ALERT_FAILURE_CATEGORY} else 'alert' + + +def _get_workflow_alert_delivery(notification): + """Return the delivery style, defaulting to popup so legacy alerts still interrupt.""" + metadata = notification.get('metadata') or {} + delivery = str(metadata.get('delivery') or '').strip().lower() + return delivery if delivery in {'notify_only', 'popup'} else 'popup' + + def _get_notification_type_config(notification): notification_type = notification.get('notification_type') if notification_type == WORKFLOW_ALERT_NOTIFICATION_TYPE: - return WORKFLOW_ALERT_PRIORITY_CONFIG.get( + priority_config = WORKFLOW_ALERT_PRIORITY_CONFIG.get( _get_workflow_alert_priority(notification), NOTIFICATION_TYPES[WORKFLOW_ALERT_NOTIFICATION_TYPE], ) + if _get_workflow_alert_category(notification) == WORKFLOW_ALERT_FAILURE_CATEGORY: + return { + 'icon': WORKFLOW_ALERT_FAILURE_ICON, + 'color': priority_config.get('color', 'danger'), + } + return priority_config return NOTIFICATION_TYPES.get( notification_type, @@ -700,6 +732,8 @@ def get_user_notifications(user_id, page=1, per_page=20, include_read=True, incl notif['type_config'] = _get_notification_type_config(notif) if notif.get('notification_type') == WORKFLOW_ALERT_NOTIFICATION_TYPE: notif['priority'] = _get_workflow_alert_priority(notif) + notif['category'] = _get_workflow_alert_category(notif) + notif['delivery'] = _get_workflow_alert_delivery(notif) filtered_notifications.append(notif) @@ -825,11 +859,16 @@ def get_unread_workflow_priority_notifications(user_id, limit=5): continue if user_id in notification.get('read_by', []): continue + # Quiet severities stay in the notification bell instead of interrupting. + if _get_workflow_alert_delivery(notification) == WORKFLOW_ALERT_DELIVERY_NOTIFY_ONLY: + continue notification['message'] = _get_notification_display_message(notification) notification['is_read'] = False notification['is_dismissed'] = False notification['priority'] = _get_workflow_alert_priority(notification) + notification['category'] = _get_workflow_alert_category(notification) + notification['delivery'] = _get_workflow_alert_delivery(notification) notification['type_config'] = _get_notification_type_config(notification) unread_notifications.append(notification) diff --git a/application/single_app/functions_personal_workflows.py b/application/single_app/functions_personal_workflows.py index 00c51bf21..3aaa75083 100644 --- a/application/single_app/functions_personal_workflows.py +++ b/application/single_app/functions_personal_workflows.py @@ -36,6 +36,7 @@ from functions_global_agents import get_global_agents from functions_personal_agents import get_personal_agents from functions_settings import get_settings, get_user_settings, normalize_model_endpoints +from functions_workflow_alerts import normalize_workflow_alert_settings WORKFLOW_TRIGGER_TYPES = {'manual', 'interval', 'file_sync'} @@ -720,9 +721,12 @@ def save_personal_workflow(user_id, workflow_data, actor_user_id=None): ), default=False, ) if url_access_enabled else False - alert_priority = _normalize_alert_priority( - workflow_data.get('alert_priority', (existing_workflow or {}).get('alert_priority', 'none')) + alert_settings = normalize_workflow_alert_settings( + workflow_data, + existing_workflow=existing_workflow, + task_ids=[task.get('id') for task in tasks], ) + alert_priority = alert_settings['alert_priority'] error_handling = _normalize_workflow_error_handling(workflow_data, existing_workflow=existing_workflow) default_chat_capabilities_enabled = ( (existing_workflow or {}).get('chat_capabilities_enabled', False) @@ -798,6 +802,9 @@ def save_personal_workflow(user_id, workflow_data, actor_user_id=None): 'URL Access authorized at', ) if url_access_authorized else '', 'alert_priority': alert_priority, + 'alert_mode': alert_settings['alert_mode'], + 'alert_rules': alert_settings['alert_rules'], + 'alert_evaluation': alert_settings['alert_evaluation'], 'schedule': schedule, 'document_action': document_action, 'analyze': analyze, diff --git a/application/single_app/functions_simplechat_operations.py b/application/single_app/functions_simplechat_operations.py index 1237c3cea..03da3a6e5 100644 --- a/application/single_app/functions_simplechat_operations.py +++ b/application/single_app/functions_simplechat_operations.py @@ -117,6 +117,7 @@ "create_personal_conversation": "create_personal_conversation", "create_personal_workflow": "create_personal_workflow", "create_personal_collaboration_conversation": "create_personal_collaboration_conversation", + "raise_workflow_alert": "raise_workflow_alert", } SIMPLECHAT_CAPABILITY_DEFINITIONS = [ { @@ -191,12 +192,24 @@ "label": "Create Personal Collaborative Conversations", "description": "Create a personal collaborative conversation and invite other users.", }, + { + "key": "raise_workflow_alert", + "function_name": SIMPLECHAT_CAPABILITY_TO_FUNCTION["raise_workflow_alert"], + "label": "Raise Workflow Alerts", + "description": "Raise an alert signal during a workflow run so the workflow's alert rules can notify the owner.", + # Opt-in: an agent that can create notifications should be granted that + # deliberately rather than inheriting it when the capability was added. + "default_enabled": False, + }, ] CONVERSATION_ACCESS_ERROR = "Conversation not found or access denied" def get_default_simplechat_capabilities() -> Dict[str, bool]: - return {definition["key"]: True for definition in SIMPLECHAT_CAPABILITY_DEFINITIONS} + return { + definition["key"]: bool(definition.get("default_enabled", True)) + for definition in SIMPLECHAT_CAPABILITY_DEFINITIONS + } def normalize_simplechat_capabilities(raw_capabilities: Any = None) -> Dict[str, bool]: @@ -952,6 +965,48 @@ def create_personal_workflow_for_current_user( } +def raise_workflow_alert_for_current_user( + severity: str = "medium", + title: str = "", + reason: str = "", + signal_name: str = "", +) -> Dict[str, Any]: + """Raise an alert signal for the workflow run the caller is executing inside. + + The signal is only recorded while a workflow run is active. Outside a run the + call is refused so a normal chat cannot fabricate a workflow notification. + """ + # Imported lazily because functions_workflow_runner imports this module. + from functions_workflow_runner import ( + is_workflow_alert_signal_scope_active, + record_workflow_alert_signal, + ) + + if not is_workflow_alert_signal_scope_active(): + raise PermissionError( + "Workflow alerts can only be raised while a workflow run is executing." + ) + + signal = record_workflow_alert_signal( + severity=severity, + title=title, + reason=reason, + signal_name=signal_name, + ) + if not signal: + raise PermissionError( + "Workflow alerts can only be raised while a workflow run is executing." + ) + + return { + "signal": signal, + "message": ( + f"Recorded a {signal.get('severity')} workflow alert signal. " + "The workflow's alert rules decide whether it notifies the owner." + ), + } + + def add_conversation_message_for_current_user( conversation_id: str, content: str, diff --git a/application/single_app/functions_workflow_activity.py b/application/single_app/functions_workflow_activity.py index 15852368a..45fc0323a 100644 --- a/application/single_app/functions_workflow_activity.py +++ b/application/single_app/functions_workflow_activity.py @@ -4,6 +4,8 @@ from datetime import datetime, timezone +from functions_workflow_alerts import describe_alert_condition, resolve_workflow_alert_config + def _normalize_text(value): return str(value or '').strip() @@ -49,6 +51,7 @@ def _serialize_workflow(workflow): if not isinstance(workflow, dict): return None + alert_config = resolve_workflow_alert_config(workflow) return { 'id': workflow.get('id'), 'name': workflow.get('name'), @@ -56,6 +59,19 @@ def _serialize_workflow(workflow): 'runner_type': workflow.get('runner_type'), 'trigger_type': workflow.get('trigger_type'), 'alert_priority': workflow.get('alert_priority'), + 'alert_mode': alert_config.get('alert_mode'), + 'alert_rules': [ + { + 'id': rule.get('id'), + 'name': rule.get('name'), + 'enabled': rule.get('enabled', True), + 'severity': rule.get('severity'), + 'delivery': rule.get('delivery'), + 'condition_summary': describe_alert_condition(rule.get('condition')), + } + for rule in alert_config.get('alert_rules') or [] + if isinstance(rule, dict) + ], 'conversation_id': workflow.get('conversation_id'), } @@ -95,6 +111,7 @@ def _serialize_run(run_record): 'agent_display_name': run_record.get('agent_display_name'), 'response_preview': run_record.get('response_preview'), 'error': run_record.get('error'), + 'alert_decision': run_record.get('alert_decision') or {}, } diff --git a/application/single_app/functions_workflow_alerts.py b/application/single_app/functions_workflow_alerts.py new file mode 100644 index 000000000..e6ae97622 --- /dev/null +++ b/application/single_app/functions_workflow_alerts.py @@ -0,0 +1,1078 @@ +# functions_workflow_alerts.py + +""" +Workflow alert rule evaluation. + +Version: 0.250.213 +Implemented in: 0.250.213 + +Workflow alerts used to fire on every run because a workflow carried a single +``alert_priority`` value. This module adds a rule engine so a workflow only +notifies when a declared condition is actually met, and so the severity of the +resulting notification reflects which condition matched. + +The module deliberately avoids importing the workflow runner or the workflow +CRUD helpers so it can be shared by the personal save path, the group save path +and the runner without creating an import cycle. Model-evaluated conditions are +resolved through a caller-supplied callable rather than a direct model client +for the same reason. +""" + +import json +import logging +import re +import uuid + +from functions_appinsights import log_event + + +# Severity ladder, ordered from quietest to loudest. +WORKFLOW_ALERT_SEVERITY_ORDER = ('info', 'low', 'medium', 'high', 'critical') +WORKFLOW_ALERT_SEVERITIES = set(WORKFLOW_ALERT_SEVERITY_ORDER) +WORKFLOW_ALERT_DEFAULT_SEVERITY = 'medium' + +WORKFLOW_ALERT_MODES = {'off', 'every_run', 'rules'} +WORKFLOW_ALERT_CATEGORIES = {'alert', 'failure'} +WORKFLOW_ALERT_DELIVERIES = {'default', 'notify_only', 'popup'} +WORKFLOW_ALERT_RESOLVED_DELIVERIES = {'notify_only', 'popup'} +WORKFLOW_ALERT_LEGACY_PRIORITIES = {'none', 'low', 'medium', 'high'} + +# Quiet severities land in the notification bell; louder ones interrupt with the modal. +WORKFLOW_ALERT_SEVERITY_DELIVERY = { + 'info': 'notify_only', + 'low': 'notify_only', + 'medium': 'popup', + 'high': 'popup', + 'critical': 'popup', +} + +WORKFLOW_ALERT_CONDITION_TYPES = { + 'run_status', + 'task_status', + 'text_match', + 'file_sync', + 'no_output', + 'model_evaluation', + 'agent_signal', +} +WORKFLOW_ALERT_RUN_STATUSES = {'completed', 'failed', 'cancelled', 'completed_with_task_errors'} +WORKFLOW_ALERT_TASK_STATUSES = {'succeeded', 'failed'} +WORKFLOW_ALERT_TEXT_MATCH_MODES = {'contains_any', 'contains_all', 'not_contains', 'regex'} +WORKFLOW_ALERT_FILE_SYNC_OUTCOMES = {'changes_found', 'no_changes', 'sync_failed'} +WORKFLOW_ALERT_SCOPE_TYPES = {'final', 'task', 'any_task'} +WORKFLOW_ALERT_EVALUATION_ERROR_MODES = {'skip', 'alert'} + +# Conditions whose match means the run itself went wrong rather than the run +# finding something noteworthy. +WORKFLOW_ALERT_FAILURE_RUN_STATUSES = {'failed', 'completed_with_task_errors'} + +WORKFLOW_ALERT_MAX_RULES = 20 +WORKFLOW_ALERT_RULE_NAME_MAX_LENGTH = 120 +WORKFLOW_ALERT_MAX_TEXT_VALUES = 25 +WORKFLOW_ALERT_TEXT_VALUE_MAX_LENGTH = 400 +WORKFLOW_ALERT_REGEX_MAX_LENGTH = 200 +WORKFLOW_ALERT_EVALUATION_PROMPT_MAX_LENGTH = 2000 +WORKFLOW_ALERT_EVALUATION_TEXT_LIMIT = 12000 +WORKFLOW_ALERT_MATCH_TEXT_LIMIT = 20000 +WORKFLOW_ALERT_REASON_MAX_LENGTH = 240 + +# Rejects the classic catastrophic-backtracking shape, a quantified group that +# itself contains a quantifier, such as ``(a+)+`` or ``(\d*)*``. +_NESTED_QUANTIFIER_PATTERN = re.compile(r'\([^)]*[*+][^)]*\)\s*[*+]') + +WORKFLOW_ALERT_CONDITION_LABELS = { + 'run_status': 'Run status', + 'task_status': 'Task status', + 'text_match': 'Output text', + 'file_sync': 'File Sync result', + 'no_output': 'No output produced', + 'model_evaluation': 'Model evaluated condition', + 'agent_signal': 'Agent raised alert', +} + + +def normalize_alert_severity(value, default=WORKFLOW_ALERT_DEFAULT_SEVERITY): + """Return a supported severity, falling back to the supplied default.""" + normalized = str(value or '').strip().lower() + if normalized in WORKFLOW_ALERT_SEVERITIES: + return normalized + return default + + +def get_alert_severity_rank(severity): + """Return the ordinal position of a severity so severities can be compared.""" + normalized = normalize_alert_severity(severity) + return WORKFLOW_ALERT_SEVERITY_ORDER.index(normalized) + + +def get_default_alert_delivery(severity): + """Return the delivery style a severity uses when the rule does not override it.""" + return WORKFLOW_ALERT_SEVERITY_DELIVERY.get( + normalize_alert_severity(severity), + 'popup', + ) + + +def resolve_alert_delivery(severity, delivery): + """Resolve a stored delivery value into ``notify_only`` or ``popup``.""" + normalized = str(delivery or 'default').strip().lower() + if normalized in WORKFLOW_ALERT_RESOLVED_DELIVERIES: + return normalized + return get_default_alert_delivery(severity) + + +def _truncate(text, limit): + normalized = str(text or '') + if len(normalized) <= limit: + return normalized + return normalized[:limit].rstrip() + '…' + + +def _normalize_reason(text): + return _truncate(' '.join(str(text or '').split()), WORKFLOW_ALERT_REASON_MAX_LENGTH) + + +def _normalize_legacy_alert_priority(value): + normalized = str(value or 'none').strip().lower() or 'none' + if normalized not in WORKFLOW_ALERT_LEGACY_PRIORITIES: + raise ValueError('Alert priority must be none, low, medium, or high.') + return normalized + + +def validate_alert_regex(pattern): + """Compile a user supplied regex, rejecting oversized or backtracking-prone patterns.""" + normalized = str(pattern or '').strip() + if not normalized: + raise ValueError('Alert rule regex pattern is required.') + if len(normalized) > WORKFLOW_ALERT_REGEX_MAX_LENGTH: + raise ValueError( + f'Alert rule regex pattern must be {WORKFLOW_ALERT_REGEX_MAX_LENGTH} characters or fewer.' + ) + if _NESTED_QUANTIFIER_PATTERN.search(normalized): + raise ValueError('Alert rule regex pattern uses nested quantifiers that are not allowed.') + + try: + return re.compile(normalized, re.IGNORECASE | re.MULTILINE) + except re.error as exc: + raise ValueError(f'Alert rule regex pattern is invalid: {exc}') from exc + + +def _normalize_string_list(values, field_name, max_items, max_length): + if isinstance(values, str): + values = [values] + if not isinstance(values, list): + raise ValueError(f'{field_name} must be a list of text values.') + + normalized_values = [] + for raw_value in values: + normalized_value = str(raw_value or '').strip() + if not normalized_value: + continue + if len(normalized_value) > max_length: + raise ValueError(f'{field_name} entries must be {max_length} characters or fewer.') + if normalized_value not in normalized_values: + normalized_values.append(normalized_value) + + if not normalized_values: + raise ValueError(f'{field_name} requires at least one value.') + if len(normalized_values) > max_items: + raise ValueError(f'{field_name} supports up to {max_items} values.') + return normalized_values + + +def _normalize_status_list(values, field_name, allowed_statuses): + if isinstance(values, str): + values = [values] + if not isinstance(values, list): + raise ValueError(f'{field_name} must be a list.') + + normalized_values = [] + for raw_value in values: + normalized_value = str(raw_value or '').strip().lower() + if not normalized_value: + continue + if normalized_value not in allowed_statuses: + allowed_text = ', '.join(sorted(allowed_statuses)) + raise ValueError(f'{field_name} must be one of: {allowed_text}.') + if normalized_value not in normalized_values: + normalized_values.append(normalized_value) + + if not normalized_values: + raise ValueError(f'{field_name} requires at least one value.') + return normalized_values + + +def _normalize_alert_rule_scope(raw_scope, task_ids=None): + raw_scope = raw_scope if isinstance(raw_scope, dict) else {} + scope_type = str(raw_scope.get('type') or 'final').strip().lower() or 'final' + if scope_type not in WORKFLOW_ALERT_SCOPE_TYPES: + raise ValueError('Alert rule scope must be final, task, or any_task.') + + task_id = str(raw_scope.get('task_id') or '').strip() + if scope_type != 'task': + return {'type': scope_type, 'task_id': ''} + + if not task_id: + raise ValueError('Alert rules scoped to a task must select a task.') + if task_ids is not None and task_id not in set(task_ids): + raise ValueError('Alert rule references a task that is not part of this workflow.') + return {'type': scope_type, 'task_id': task_id} + + +def _normalize_alert_condition(raw_condition, task_ids=None): + raw_condition = raw_condition if isinstance(raw_condition, dict) else {} + condition_type = str(raw_condition.get('type') or '').strip().lower() + if condition_type not in WORKFLOW_ALERT_CONDITION_TYPES: + allowed_text = ', '.join(sorted(WORKFLOW_ALERT_CONDITION_TYPES)) + raise ValueError(f'Alert rule condition type must be one of: {allowed_text}.') + + if condition_type == 'run_status': + return { + 'type': condition_type, + 'statuses': _normalize_status_list( + raw_condition.get('statuses'), + 'Alert rule run statuses', + WORKFLOW_ALERT_RUN_STATUSES, + ), + } + + if condition_type == 'task_status': + return { + 'type': condition_type, + 'statuses': _normalize_status_list( + raw_condition.get('statuses'), + 'Alert rule task statuses', + WORKFLOW_ALERT_TASK_STATUSES, + ), + } + + if condition_type == 'text_match': + match_mode = str(raw_condition.get('mode') or 'contains_any').strip().lower() or 'contains_any' + if match_mode not in WORKFLOW_ALERT_TEXT_MATCH_MODES: + raise ValueError('Alert rule text match mode must be contains_any, contains_all, not_contains, or regex.') + + if match_mode == 'regex': + pattern = str(raw_condition.get('pattern') or '').strip() + validate_alert_regex(pattern) + return { + 'type': condition_type, + 'mode': match_mode, + 'pattern': pattern, + 'values': [], + 'case_sensitive': False, + } + + return { + 'type': condition_type, + 'mode': match_mode, + 'pattern': '', + 'values': _normalize_string_list( + raw_condition.get('values'), + 'Alert rule match values', + WORKFLOW_ALERT_MAX_TEXT_VALUES, + WORKFLOW_ALERT_TEXT_VALUE_MAX_LENGTH, + ), + 'case_sensitive': bool(raw_condition.get('case_sensitive')), + } + + if condition_type == 'file_sync': + outcome = str(raw_condition.get('outcome') or '').strip().lower() + if outcome not in WORKFLOW_ALERT_FILE_SYNC_OUTCOMES: + raise ValueError('Alert rule File Sync outcome must be changes_found, no_changes, or sync_failed.') + return {'type': condition_type, 'outcome': outcome} + + if condition_type == 'no_output': + return {'type': condition_type} + + if condition_type == 'model_evaluation': + prompt = str(raw_condition.get('prompt') or '').strip() + if not prompt: + raise ValueError('Alert rule model evaluation requires a condition to evaluate.') + if len(prompt) > WORKFLOW_ALERT_EVALUATION_PROMPT_MAX_LENGTH: + raise ValueError( + f'Alert rule model evaluation condition must be ' + f'{WORKFLOW_ALERT_EVALUATION_PROMPT_MAX_LENGTH} characters or fewer.' + ) + return {'type': condition_type, 'prompt': prompt} + + signal_name = str(raw_condition.get('signal_name') or '').strip() + if len(signal_name) > WORKFLOW_ALERT_RULE_NAME_MAX_LENGTH: + raise ValueError( + f'Alert rule signal name must be {WORKFLOW_ALERT_RULE_NAME_MAX_LENGTH} characters or fewer.' + ) + return { + 'type': condition_type, + 'signal_name': signal_name, + 'min_severity': normalize_alert_severity(raw_condition.get('min_severity'), default='info'), + } + + +def describe_alert_condition(condition): + """Return a short human readable description of a normalized condition.""" + condition = condition if isinstance(condition, dict) else {} + condition_type = str(condition.get('type') or '').strip().lower() + label = WORKFLOW_ALERT_CONDITION_LABELS.get(condition_type, 'Condition') + + if condition_type == 'run_status': + return f"{label} is {', '.join(condition.get('statuses') or [])}" + if condition_type == 'task_status': + return f"{label} is {', '.join(condition.get('statuses') or [])}" + if condition_type == 'text_match': + if condition.get('mode') == 'regex': + return f"{label} matches /{condition.get('pattern')}/" + mode_labels = { + 'contains_any': 'contains any of', + 'contains_all': 'contains all of', + 'not_contains': 'does not contain', + } + mode_label = mode_labels.get(condition.get('mode'), 'contains') + return f"{label} {mode_label} {', '.join(condition.get('values') or [])}" + if condition_type == 'file_sync': + return f"{label} is {condition.get('outcome')}" + if condition_type == 'model_evaluation': + return f"{label}: {condition.get('prompt')}" + if condition_type == 'agent_signal': + signal_name = condition.get('signal_name') + if signal_name: + return f"{label} named {signal_name}" + return label + return label + + +def normalize_alert_rule(raw_rule, task_ids=None, index=0): + """Normalize and validate a single alert rule.""" + if not isinstance(raw_rule, dict): + raise ValueError(f'Alert rule {index + 1} is invalid.') + + condition = _normalize_alert_condition(raw_rule.get('condition'), task_ids=task_ids) + scope = _normalize_alert_rule_scope(raw_rule.get('scope'), task_ids=task_ids) + + name = str(raw_rule.get('name') or '').strip() + if not name: + name = describe_alert_condition(condition) + if len(name) > WORKFLOW_ALERT_RULE_NAME_MAX_LENGTH: + raise ValueError( + f'Alert rule name must be {WORKFLOW_ALERT_RULE_NAME_MAX_LENGTH} characters or fewer.' + ) + + delivery = str(raw_rule.get('delivery') or 'default').strip().lower() or 'default' + if delivery not in WORKFLOW_ALERT_DELIVERIES: + raise ValueError('Alert rule delivery must be default, notify_only, or popup.') + + severity = str(raw_rule.get('severity') or WORKFLOW_ALERT_DEFAULT_SEVERITY).strip().lower() + if severity not in WORKFLOW_ALERT_SEVERITIES: + allowed_text = ', '.join(WORKFLOW_ALERT_SEVERITY_ORDER) + raise ValueError(f'Alert rule severity must be one of: {allowed_text}.') + + enabled = raw_rule.get('enabled', True) + if isinstance(enabled, str): + enabled = enabled.strip().lower() in {'1', 'true', 'yes', 'on'} + + return { + 'id': str(raw_rule.get('id') or '').strip() or str(uuid.uuid4()), + 'name': name, + 'enabled': bool(enabled), + 'severity': severity, + 'delivery': delivery, + 'scope': scope, + 'condition': condition, + 'order': index + 1, + } + + +def normalize_alert_rules(raw_rules, task_ids=None): + """Normalize a list of alert rules, enforcing the rule limit and unique ids.""" + if raw_rules is None: + return [] + if not isinstance(raw_rules, list): + raise ValueError('Alert rules must be a list.') + if len(raw_rules) > WORKFLOW_ALERT_MAX_RULES: + raise ValueError(f'Workflows support up to {WORKFLOW_ALERT_MAX_RULES} alert rules.') + + normalized_rules = [] + seen_rule_ids = set() + for index, raw_rule in enumerate(raw_rules): + normalized_rule = normalize_alert_rule(raw_rule, task_ids=task_ids, index=index) + if normalized_rule['id'] in seen_rule_ids: + normalized_rule['id'] = str(uuid.uuid4()) + seen_rule_ids.add(normalized_rule['id']) + normalized_rules.append(normalized_rule) + return normalized_rules + + +def _normalize_alert_evaluation(raw_evaluation): + raw_evaluation = raw_evaluation if isinstance(raw_evaluation, dict) else {} + on_error = str(raw_evaluation.get('on_error') or 'skip').strip().lower() or 'skip' + if on_error not in WORKFLOW_ALERT_EVALUATION_ERROR_MODES: + raise ValueError('Alert evaluation error handling must be skip or alert.') + return {'on_error': on_error} + + +def build_legacy_alert_rules(alert_priority): + """Build the editable rules that reproduce the legacy alert-on-every-run behavior. + + Legacy alerts always opened the modal regardless of priority, so the migrated + rules pin ``delivery`` to ``popup`` instead of inheriting the severity default. + """ + normalized_priority = _normalize_legacy_alert_priority(alert_priority) + if normalized_priority == 'none': + return [] + + return [ + { + 'id': 'legacy-run-failed', + 'name': 'Run failed', + 'enabled': True, + 'severity': 'high', + 'delivery': 'popup', + 'scope': {'type': 'final', 'task_id': ''}, + 'condition': {'type': 'run_status', 'statuses': ['failed']}, + 'order': 1, + }, + { + 'id': 'legacy-run-completed', + 'name': 'Run completed', + 'enabled': True, + 'severity': normalized_priority, + 'delivery': 'popup', + 'scope': {'type': 'final', 'task_id': ''}, + 'condition': {'type': 'run_status', 'statuses': ['completed']}, + 'order': 2, + }, + ] + + +def resolve_workflow_alert_config(workflow): + """Return the effective alert configuration for a workflow document. + + Workflows saved before alert rules existed only carry ``alert_priority``. They + are migrated on read so they keep alerting exactly as before without needing a + stored data migration. + """ + workflow = workflow if isinstance(workflow, dict) else {} + stored_rules = workflow.get('alert_rules') if isinstance(workflow.get('alert_rules'), list) else None + stored_mode = str(workflow.get('alert_mode') or '').strip().lower() + alert_priority = str(workflow.get('alert_priority') or 'none').strip().lower() + if alert_priority not in WORKFLOW_ALERT_LEGACY_PRIORITIES: + alert_priority = 'none' + + if stored_mode in WORKFLOW_ALERT_MODES: + mode = stored_mode + rules = stored_rules or [] + elif stored_rules: + mode = 'rules' + rules = stored_rules + elif alert_priority != 'none': + mode = 'rules' + rules = build_legacy_alert_rules(alert_priority) + else: + mode = 'off' + rules = [] + + evaluation = workflow.get('alert_evaluation') if isinstance(workflow.get('alert_evaluation'), dict) else {} + return { + 'alert_mode': mode, + 'alert_priority': alert_priority, + 'alert_rules': rules, + 'alert_evaluation': { + 'on_error': str(evaluation.get('on_error') or 'skip').strip().lower() or 'skip', + }, + } + + +def normalize_workflow_alert_settings(workflow_data, existing_workflow=None, task_ids=None): + """Normalize the alert fields for a workflow save request. + + Returns a dict with ``alert_mode``, ``alert_priority``, ``alert_rules`` and + ``alert_evaluation`` ready to persist on the workflow document. + """ + workflow_data = workflow_data if isinstance(workflow_data, dict) else {} + existing_workflow = existing_workflow if isinstance(existing_workflow, dict) else {} + existing_config = resolve_workflow_alert_config(existing_workflow) + + alert_priority = _normalize_legacy_alert_priority( + workflow_data.get('alert_priority', existing_config.get('alert_priority', 'none')) + ) + + if 'alert_rules' in workflow_data: + alert_rules = normalize_alert_rules(workflow_data.get('alert_rules'), task_ids=task_ids) + else: + alert_rules = normalize_alert_rules(existing_config.get('alert_rules'), task_ids=None) + + if 'alert_mode' in workflow_data: + alert_mode = str(workflow_data.get('alert_mode') or '').strip().lower() + if alert_mode not in WORKFLOW_ALERT_MODES: + raise ValueError('Alert mode must be off, every_run, or rules.') + elif 'alert_rules' in workflow_data: + alert_mode = 'rules' if alert_rules else 'off' + else: + alert_mode = existing_config.get('alert_mode') or 'off' + if alert_mode == 'off' and alert_priority != 'none' and not alert_rules: + # Clients that predate alert rules only send alert_priority, so keep + # them alerting by materializing the equivalent editable rules. + alert_rules = normalize_alert_rules(build_legacy_alert_rules(alert_priority)) + alert_mode = 'rules' + + if alert_mode == 'rules' and not alert_rules: + raise ValueError('Add at least one alert rule or choose a different alert mode.') + if alert_mode == 'every_run' and alert_priority == 'none': + raise ValueError('Select an alert priority or choose a different alert mode.') + + if 'alert_evaluation' in workflow_data: + alert_evaluation = _normalize_alert_evaluation(workflow_data.get('alert_evaluation')) + else: + alert_evaluation = _normalize_alert_evaluation(existing_config.get('alert_evaluation')) + + return { + 'alert_mode': alert_mode, + 'alert_priority': alert_priority, + 'alert_rules': alert_rules, + 'alert_evaluation': alert_evaluation, + } + + +def normalize_agent_alert_signal(raw_signal): + """Normalize an alert signal raised by an agent during a workflow run.""" + raw_signal = raw_signal if isinstance(raw_signal, dict) else {} + title = _truncate(' '.join(str(raw_signal.get('title') or '').split()), WORKFLOW_ALERT_RULE_NAME_MAX_LENGTH) + return { + 'severity': normalize_alert_severity(raw_signal.get('severity'), default='medium'), + 'signal_name': str(raw_signal.get('signal_name') or '').strip(), + 'title': title, + 'reason': _normalize_reason(raw_signal.get('reason')), + } + + +def build_workflow_alert_facts(workflow, run_record, execution_result=None): + """Collect everything the rule engine needs from a finished run.""" + workflow = workflow if isinstance(workflow, dict) else {} + run_record = run_record if isinstance(run_record, dict) else {} + execution_result = execution_result if isinstance(execution_result, dict) else {} + + run_status = str(run_record.get('status') or '').strip().lower() or 'completed' + task_results = execution_result.get('task_results') + if not isinstance(task_results, list): + task_results = run_record.get('task_results') if isinstance(run_record.get('task_results'), list) else [] + + task_outputs = [] + for index, task_result in enumerate(task_results): + if not isinstance(task_result, dict): + continue + task = task_result.get('task') if isinstance(task_result.get('task'), dict) else {} + nested_result = task_result.get('result') if isinstance(task_result.get('result'), dict) else {} + task_outputs.append({ + 'task_id': str(task.get('id') or '').strip(), + 'task_name': str(task.get('name') or f'Task {index + 1}').strip(), + 'status': str(task_result.get('status') or '').strip().lower(), + 'output': str(nested_result.get('reply') or '').strip(), + 'error': str(task_result.get('error') or '').strip(), + }) + + try: + task_error_count = int(execution_result.get('task_error_count') or run_record.get('task_error_count') or 0) + except (TypeError, ValueError): + task_error_count = 0 + if not task_error_count: + task_error_count = sum(1 for item in task_outputs if item.get('status') == 'failed') + + effective_statuses = {run_status} + if run_status == 'completed' and task_error_count: + effective_statuses.add('completed_with_task_errors') + + file_sync = run_record.get('file_sync') if isinstance(run_record.get('file_sync'), dict) else {} + if not file_sync and isinstance(execution_result.get('file_sync'), dict): + file_sync = execution_result.get('file_sync') + + raw_signals = execution_result.get('agent_alert_signals') + if not isinstance(raw_signals, list): + raw_signals = [] + + final_output = str(execution_result.get('reply') or run_record.get('response_preview') or '').strip() + + return { + 'workflow_id': str(workflow.get('id') or '').strip(), + 'workflow_name': str(workflow.get('name') or 'Workflow').strip() or 'Workflow', + 'run_status': run_status, + 'effective_run_statuses': effective_statuses, + 'success': bool(run_record.get('success')), + 'error': str(run_record.get('error') or '').strip(), + 'final_output': final_output, + 'task_outputs': task_outputs, + 'task_error_count': task_error_count, + 'file_sync': file_sync, + 'file_sync_outcome': _resolve_file_sync_outcome(file_sync), + 'agent_signals': [normalize_agent_alert_signal(signal) for signal in raw_signals], + 'trigger_source': str(run_record.get('trigger_source') or 'manual').strip() or 'manual', + } + + +def _resolve_file_sync_outcome(file_sync): + file_sync = file_sync if isinstance(file_sync, dict) else {} + if not file_sync: + return '' + if file_sync.get('error') or str(file_sync.get('status') or '').strip().lower() == 'failed': + return 'sync_failed' + + changed_documents = file_sync.get('changed_documents') + changed_count = len(changed_documents) if isinstance(changed_documents, list) else 0 + if not changed_count: + try: + changed_count = int(file_sync.get('changed_document_count') or 0) + except (TypeError, ValueError): + changed_count = 0 + return 'changes_found' if changed_count else 'no_changes' + + +def _collect_scoped_texts(facts, scope): + scope = scope if isinstance(scope, dict) else {} + scope_type = str(scope.get('type') or 'final').strip().lower() + task_outputs = facts.get('task_outputs') or [] + + if scope_type == 'task': + task_id = str(scope.get('task_id') or '').strip() + return [ + (item.get('task_name') or 'Task', item.get('output') or '') + for item in task_outputs + if item.get('task_id') == task_id + ] + + if scope_type == 'any_task': + return [(item.get('task_name') or 'Task', item.get('output') or '') for item in task_outputs] + + return [('Final output', facts.get('final_output') or '')] + + +def _collect_scoped_tasks(facts, scope): + scope = scope if isinstance(scope, dict) else {} + scope_type = str(scope.get('type') or 'final').strip().lower() + task_outputs = facts.get('task_outputs') or [] + if scope_type == 'task': + task_id = str(scope.get('task_id') or '').strip() + return [item for item in task_outputs if item.get('task_id') == task_id] + return list(task_outputs) + + +def _evaluate_run_status_condition(condition, facts): + statuses = set(condition.get('statuses') or []) + matched_statuses = sorted(statuses & set(facts.get('effective_run_statuses') or set())) + if not matched_statuses: + return None + + category = 'failure' if set(matched_statuses) & WORKFLOW_ALERT_FAILURE_RUN_STATUSES else 'alert' + return { + 'reason': f"Run status is {', '.join(matched_statuses)}", + 'category': category, + } + + +def _evaluate_task_status_condition(condition, facts, scope): + statuses = set(condition.get('statuses') or []) + matched_tasks = [ + item for item in _collect_scoped_tasks(facts, scope) + if item.get('status') in statuses + ] + if not matched_tasks: + return None + + task_names = ', '.join(item.get('task_name') or 'Task' for item in matched_tasks[:3]) + matched_statuses = {item.get('status') for item in matched_tasks} + return { + 'reason': f"Task status {', '.join(sorted(matched_statuses))} for {task_names}", + 'category': 'failure' if 'failed' in matched_statuses else 'alert', + } + + +def _evaluate_text_match_condition(condition, facts, scope): + candidates = [ + (label, _truncate(text, WORKFLOW_ALERT_MATCH_TEXT_LIMIT)) + for label, text in _collect_scoped_texts(facts, scope) + ] + candidates = [(label, text) for label, text in candidates if text] + match_mode = str(condition.get('mode') or 'contains_any').strip().lower() + + if match_mode == 'regex': + compiled_pattern = validate_alert_regex(condition.get('pattern')) + for label, text in candidates: + match = compiled_pattern.search(text) + if match: + return { + 'reason': f"{label} matched pattern /{condition.get('pattern')}/", + 'category': 'alert', + } + return None + + case_sensitive = bool(condition.get('case_sensitive')) + values = list(condition.get('values') or []) + comparison_values = values if case_sensitive else [value.lower() for value in values] + + if match_mode == 'not_contains': + if not candidates: + return { + 'reason': 'No output contained the configured text', + 'category': 'alert', + } + for _label, text in candidates: + haystack = text if case_sensitive else text.lower() + if any(value in haystack for value in comparison_values): + return None + return { + 'reason': f"No output contained {', '.join(values[:3])}", + 'category': 'alert', + } + + for label, text in candidates: + haystack = text if case_sensitive else text.lower() + matched_values = [ + values[index] for index, value in enumerate(comparison_values) if value in haystack + ] + if match_mode == 'contains_all' and len(matched_values) != len(comparison_values): + continue + if not matched_values: + continue + return { + 'reason': f"{label} contains {', '.join(matched_values[:3])}", + 'category': 'alert', + } + return None + + +def _evaluate_file_sync_condition(condition, facts): + outcome = str(condition.get('outcome') or '').strip().lower() + if not outcome or outcome != facts.get('file_sync_outcome'): + return None + + outcome_labels = { + 'changes_found': 'File Sync found changed documents', + 'no_changes': 'File Sync found no changed documents', + 'sync_failed': 'File Sync failed', + } + return { + 'reason': outcome_labels.get(outcome, 'File Sync condition met'), + 'category': 'failure' if outcome == 'sync_failed' else 'alert', + } + + +def _evaluate_no_output_condition(facts, scope): + candidates = _collect_scoped_texts(facts, scope) + if any(str(text or '').strip() for _label, text in candidates): + return None + return { + 'reason': 'The run produced no output', + 'category': 'failure', + } + + +def _evaluate_agent_signal_condition(condition, facts): + signal_name = str(condition.get('signal_name') or '').strip().lower() + min_rank = get_alert_severity_rank(condition.get('min_severity') or 'info') + + for signal in facts.get('agent_signals') or []: + if signal_name and str(signal.get('signal_name') or '').strip().lower() != signal_name: + continue + if get_alert_severity_rank(signal.get('severity')) < min_rank: + continue + reason = signal.get('reason') or signal.get('title') or 'The agent raised an alert' + return { + 'reason': reason, + 'category': 'alert', + 'severity_floor': signal.get('severity'), + } + return None + + +def _evaluate_deterministic_rule(rule, facts): + condition = rule.get('condition') if isinstance(rule.get('condition'), dict) else {} + condition_type = str(condition.get('type') or '').strip().lower() + scope = rule.get('scope') if isinstance(rule.get('scope'), dict) else {} + + if condition_type == 'run_status': + return _evaluate_run_status_condition(condition, facts) + if condition_type == 'task_status': + return _evaluate_task_status_condition(condition, facts, scope) + if condition_type == 'text_match': + return _evaluate_text_match_condition(condition, facts, scope) + if condition_type == 'file_sync': + return _evaluate_file_sync_condition(condition, facts) + if condition_type == 'no_output': + return _evaluate_no_output_condition(facts, scope) + if condition_type == 'agent_signal': + return _evaluate_agent_signal_condition(condition, facts) + return None + + +def build_model_evaluation_prompt(rules, facts): + """Build the single batched prompt used to judge all model-evaluated rules.""" + condition_lines = [] + for rule in rules: + condition = rule.get('condition') if isinstance(rule.get('condition'), dict) else {} + condition_lines.append(f"- rule_id: {rule.get('id')}\n condition: {condition.get('prompt')}") + + scoped_sections = [] + for rule in rules: + for label, text in _collect_scoped_texts(facts, rule.get('scope')): + section = f'{label}:\n{text}' + if text and section not in scoped_sections: + scoped_sections.append(section) + if not scoped_sections: + scoped_sections.append('Final output:\n(no output was produced)') + + output_text = _truncate('\n\n'.join(scoped_sections), WORKFLOW_ALERT_EVALUATION_TEXT_LIMIT) + + return ( + 'You are evaluating the output of an automated workflow run against alert conditions.\n' + 'Decide, for each condition, whether it is met by the workflow output below.\n' + 'Respond with ONLY a JSON object and no markdown, using exactly this shape:\n' + '{"results": [{"rule_id": "", "matched": true, "reason": ""}]}\n\n' + f"Workflow: {facts.get('workflow_name')}\n" + f"Run status: {facts.get('run_status')}\n\n" + '--- WORKFLOW OUTPUT ---\n' + f'{output_text}\n' + '--- END WORKFLOW OUTPUT ---\n\n' + 'Conditions:\n' + f"{chr(10).join(condition_lines)}\n" + ) + + +def parse_model_evaluation_response(response_text): + """Parse the JSON verdict payload returned for model-evaluated conditions.""" + normalized_text = str(response_text or '').strip() + if not normalized_text: + raise ValueError('Model evaluation returned an empty response.') + + try: + payload = json.loads(normalized_text) + except (TypeError, ValueError): + start_index = normalized_text.find('{') + end_index = normalized_text.rfind('}') + if start_index == -1 or end_index <= start_index: + raise ValueError('Model evaluation response was not valid JSON.') + try: + payload = json.loads(normalized_text[start_index:end_index + 1]) + except (TypeError, ValueError) as exc: + raise ValueError('Model evaluation response was not valid JSON.') from exc + + if not isinstance(payload, dict): + raise ValueError('Model evaluation response was not a JSON object.') + + raw_results = payload.get('results') + if not isinstance(raw_results, list): + raise ValueError('Model evaluation response did not include a results list.') + + verdicts = {} + for raw_result in raw_results: + if not isinstance(raw_result, dict): + continue + rule_id = str(raw_result.get('rule_id') or '').strip() + if not rule_id: + continue + matched = raw_result.get('matched') + if isinstance(matched, str): + matched = matched.strip().lower() in {'1', 'true', 'yes'} + verdicts[rule_id] = { + 'matched': bool(matched), + 'reason': _normalize_reason(raw_result.get('reason')), + } + return verdicts + + +def _build_match(rule, evaluation, source='rule'): + severity = normalize_alert_severity(rule.get('severity')) + severity_floor = evaluation.get('severity_floor') + if severity_floor and get_alert_severity_rank(severity_floor) > get_alert_severity_rank(severity): + severity = normalize_alert_severity(severity_floor) + + condition = rule.get('condition') if isinstance(rule.get('condition'), dict) else {} + return { + 'rule_id': rule.get('id'), + 'rule_name': rule.get('name'), + 'severity': severity, + 'delivery': rule.get('delivery') or 'default', + 'condition_type': str(condition.get('type') or '').strip().lower(), + 'reason': _normalize_reason(evaluation.get('reason')), + 'category': evaluation.get('category') or 'alert', + 'order': rule.get('order') or 0, + 'source': source, + } + + +def _build_decision(should_alert, mode, matches=None, model_evaluation=None, evaluated_rule_count=0): + matches = list(matches or []) + model_evaluation = model_evaluation or {'used': False, 'error': '', 'rule_ids': []} + + if not should_alert or not matches: + return { + 'should_alert': False, + 'severity': '', + 'category': 'alert', + 'delivery': '', + 'mode': mode, + 'matched_rules': matches, + 'reasons': [match.get('reason') for match in matches if match.get('reason')], + 'evaluated_rule_count': evaluated_rule_count, + 'model_evaluation': model_evaluation, + } + + matches.sort(key=lambda match: (-get_alert_severity_rank(match.get('severity')), match.get('order') or 0)) + winning_match = matches[0] + severity = normalize_alert_severity(winning_match.get('severity')) + + return { + 'should_alert': True, + 'severity': severity, + 'category': winning_match.get('category') or 'alert', + 'delivery': resolve_alert_delivery(severity, winning_match.get('delivery')), + 'mode': mode, + 'winning_rule_id': winning_match.get('rule_id'), + 'winning_rule_name': winning_match.get('rule_name'), + 'matched_rules': matches, + 'reasons': [match.get('reason') for match in matches if match.get('reason')], + 'evaluated_rule_count': evaluated_rule_count, + 'model_evaluation': model_evaluation, + } + + +def evaluate_workflow_alert_rules(workflow, facts, model_evaluator=None): + """Decide whether a finished workflow run should raise an alert, and how loudly. + + ``model_evaluator`` is an optional callable taking the batched prompt text and + returning the raw model response. It is injected so this module stays free of + model client imports. + """ + facts = facts if isinstance(facts, dict) else {} + config = resolve_workflow_alert_config(workflow) + mode = config.get('alert_mode') + + if mode == 'off': + return _build_decision(False, mode) + + if mode == 'every_run': + alert_priority = config.get('alert_priority') + if alert_priority == 'none' or facts.get('run_status') == 'cancelled': + return _build_decision(False, mode) + + severity = normalize_alert_severity(alert_priority) + legacy_match = { + 'rule_id': 'every-run', + 'rule_name': 'Every run', + 'severity': severity, + 'delivery': 'popup', + 'condition_type': 'run_status', + 'reason': f"Workflow run {facts.get('run_status') or 'completed'}", + 'category': 'alert' if facts.get('success') else 'failure', + 'order': 1, + 'source': 'legacy', + } + return _build_decision(True, mode, matches=[legacy_match], evaluated_rule_count=1) + + enabled_rules = [ + rule for rule in (config.get('alert_rules') or []) + if isinstance(rule, dict) and rule.get('enabled', True) + ] + if not enabled_rules: + return _build_decision(False, mode) + + matches = [] + model_rules = [] + for rule in enabled_rules: + condition = rule.get('condition') if isinstance(rule.get('condition'), dict) else {} + if str(condition.get('type') or '').strip().lower() == 'model_evaluation': + model_rules.append(rule) + continue + try: + evaluation = _evaluate_deterministic_rule(rule, facts) + except Exception as exc: + log_event( + f'[WORKFLOW_ALERTS] Alert rule evaluation failed: {exc}', + extra={ + 'workflow_id': facts.get('workflow_id'), + 'rule_id': rule.get('id'), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + continue + if evaluation: + matches.append(_build_match(rule, evaluation)) + + best_rank = max( + (get_alert_severity_rank(match.get('severity')) for match in matches), + default=-1, + ) + # A model call cannot change the outcome when a deterministic rule already + # matched at or above the severity of every remaining model-evaluated rule. + pending_model_rules = [ + rule for rule in model_rules + if get_alert_severity_rank(rule.get('severity')) > best_rank + ] + + model_evaluation_state = { + 'used': False, + 'error': '', + 'rule_ids': [rule.get('id') for rule in pending_model_rules], + 'skipped_rule_ids': [ + rule.get('id') for rule in model_rules if rule not in pending_model_rules + ], + } + + if pending_model_rules and callable(model_evaluator): + model_evaluation_state['used'] = True + try: + prompt = build_model_evaluation_prompt(pending_model_rules, facts) + verdicts = parse_model_evaluation_response(model_evaluator(prompt)) + for rule in pending_model_rules: + verdict = verdicts.get(rule.get('id')) or {} + if verdict.get('matched'): + matches.append(_build_match( + rule, + { + 'reason': verdict.get('reason') or rule.get('name'), + 'category': 'alert', + }, + source='model_evaluation', + )) + except Exception as exc: + model_evaluation_state['error'] = str(exc) + log_event( + f'[WORKFLOW_ALERTS] Model evaluated alert conditions could not be judged: {exc}', + extra={'workflow_id': facts.get('workflow_id')}, + level=logging.WARNING, + exceptionTraceback=True, + ) + if config.get('alert_evaluation', {}).get('on_error') == 'alert': + for rule in pending_model_rules: + matches.append(_build_match( + rule, + { + 'reason': f'Alert condition could not be evaluated: {exc}', + 'category': 'failure', + }, + source='model_evaluation_error', + )) + elif pending_model_rules: + model_evaluation_state['error'] = 'No model evaluator was available for this run.' + + return _build_decision( + bool(matches), + mode, + matches=matches, + model_evaluation=model_evaluation_state, + evaluated_rule_count=len(enabled_rules), + ) + + +def summarize_alert_decision(decision): + """Return a short one line summary of an alert decision for logs and activity views.""" + decision = decision if isinstance(decision, dict) else {} + if not decision.get('should_alert'): + if decision.get('mode') == 'off': + return 'Alerts are turned off for this workflow.' + return 'No alert rule matched this run.' + + matched_names = [ + match.get('rule_name') for match in decision.get('matched_rules') or [] + if match.get('rule_name') + ] + severity = str(decision.get('severity') or '').upper() + if matched_names: + return f"{severity} alert triggered by: {', '.join(matched_names)}" + return f'{severity} alert triggered.' diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 1eab103f1..83ccfd1ad 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -13,6 +13,7 @@ import time import uuid from contextlib import contextmanager +from contextvars import ContextVar from datetime import datetime, timezone from azure.cosmos.exceptions import CosmosResourceNotFoundError @@ -155,6 +156,14 @@ build_semantic_kernel_chat_service_for_model, ) from functions_notifications import create_workflow_priority_notification +from functions_workflow_alerts import ( + build_workflow_alert_facts, + evaluate_workflow_alert_rules, + normalize_agent_alert_signal, + normalize_alert_severity, + resolve_workflow_alert_config, + summarize_alert_decision, +) from functions_personal_workflows import ( get_personal_workflow, get_personal_workflow_run, @@ -4550,12 +4559,98 @@ def _mirror_workflow_visualizations_to_created_conversations(workflow, source_as WORKFLOW_ALERT_PRIORITIES = {'low', 'medium', 'high'} +# Alert signals raised by an agent during a run are collected per run through a +# context variable so the SimpleChat plugin can hand them to the rule engine +# without threading the run through every kernel invocation. +_workflow_alert_signal_context = ContextVar('workflow_alert_signals', default=None) + + +@contextmanager +def workflow_alert_signal_scope(workflow=None, run_id=None): + """Collect agent raised alert signals for the duration of a workflow run.""" + scope = { + 'workflow_id': str((workflow or {}).get('id') or '').strip(), + 'run_id': str(run_id or '').strip(), + 'signals': [], + } + token = _workflow_alert_signal_context.set(scope) + try: + yield scope + finally: + _workflow_alert_signal_context.reset(token) + + +def is_workflow_alert_signal_scope_active(): + """Return True when the caller is running inside a workflow run.""" + return isinstance(_workflow_alert_signal_context.get(), dict) + + +def record_workflow_alert_signal(severity, title='', reason='', signal_name=''): + """Record an agent raised alert signal for the active workflow run. + + Returns the normalized signal, or None when no workflow run is active so the + caller can refuse instead of fabricating a notification. + """ + scope = _workflow_alert_signal_context.get() + if not isinstance(scope, dict): + return None + + signal = normalize_agent_alert_signal({ + 'severity': severity, + 'title': title, + 'reason': reason, + 'signal_name': signal_name, + }) + scope['signals'].append(signal) + return signal + + +def get_workflow_alert_signals(): + """Return the alert signals raised so far during the active workflow run.""" + scope = _workflow_alert_signal_context.get() + if not isinstance(scope, dict): + return [] + return list(scope.get('signals') or []) -def _normalize_workflow_alert_priority(priority): - normalized = str(priority or '').strip().lower() - if normalized not in WORKFLOW_ALERT_PRIORITIES: - return 'none' - return normalized + +def _build_workflow_alert_model_evaluator(workflow, settings=None): + """Build the callable the rule engine uses to judge model evaluated conditions. + + Returns None when no model can be resolved, in which case model evaluated + rules are reported as unevaluated instead of silently matching. + """ + try: + evaluation_settings = settings if isinstance(settings, dict) else get_settings() + client, deployment_name, _provider = _resolve_model_workflow_client(workflow, evaluation_settings) + except Exception as exc: + log_event( + f'[WORKFLOW_RUNNER] Alert condition evaluator unavailable: {exc}', + extra={'workflow_id': str((workflow or {}).get('id') or '').strip()}, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + def evaluate(prompt): + completion = client.chat.completions.create( + model=deployment_name, + messages=[ + { + 'role': 'system', + 'content': ( + 'You evaluate automated workflow output against alert conditions. ' + 'You always reply with a single JSON object and never with markdown or commentary.' + ), + }, + {'role': 'user', 'content': prompt}, + ], + temperature=0, + ) + if not getattr(completion, 'choices', None): + return '' + return _extract_message_text(completion.choices[0].message.content) + + return evaluate def _dedupe_workflow_alert_targets(targets): @@ -4969,25 +5064,53 @@ def _build_workflow_alert_success_detail(alert_title, action_plan, response_prev ) -def _build_workflow_alert_content(workflow, run_record, execution_result, priority): +def _build_workflow_alert_trigger_section(decision): + """Render the "Triggered by" section listing every rule that matched the run.""" + decision = decision if isinstance(decision, dict) else {} + matched_rules = decision.get('matched_rules') or [] + if not matched_rules: + return '' + + trigger_lines = [] + for match in matched_rules: + rule_name = _normalize_workflow_alert_text(match.get('rule_name') or 'Alert rule') + severity = str(match.get('severity') or '').strip().lower() + reason = _normalize_workflow_alert_text(match.get('reason')) + line = f'- {rule_name}' + if severity: + line = f'{line} ({severity})' + if reason: + line = f'{line}: {reason}' + trigger_lines.append(line) + + return 'Triggered by\n' + '\n'.join(trigger_lines) + + +def _build_workflow_alert_content(workflow, run_record, execution_result, priority, decision=None): execution_result = execution_result if isinstance(execution_result, dict) else {} + decision = decision if isinstance(decision, dict) else {} workflow_name = _normalize_workflow_alert_title_text(workflow.get('name') or 'Workflow') or 'Workflow' trigger_source = str(run_record.get('trigger_source') or 'manual').strip() or 'manual' success = bool(run_record.get('success')) + is_failure_alert = str(decision.get('category') or '').strip().lower() == 'failure' or not success response_preview = _strip_workflow_alert_markdown(run_record.get('response_preview') or '') reply_text = _strip_workflow_alert_markdown(execution_result.get('reply') or '') error_text = _strip_workflow_alert_markdown(run_record.get('error') or '') agent_citations = list(execution_result.get('agent_citations') or []) enrichment_labels = _build_workflow_alert_enrichment_labels(agent_citations) action_plan = _build_workflow_alert_action_plan(agent_citations) + trigger_section = _build_workflow_alert_trigger_section(decision) + winning_rule_name = _normalize_workflow_alert_title_text(decision.get('winning_rule_name') or '') alert_title = _extract_workflow_alert_title_from_citations(agent_citations) if not alert_title: alert_title = _extract_workflow_alert_event_title(reply_text or response_preview) if not alert_title: - alert_title = workflow_name + alert_title = winning_rule_name or workflow_name + + severity_label = str(decision.get('severity') or priority or '').strip().lower() or 'medium' - if success: + if not is_failure_alert: alert_summary = _build_workflow_alert_success_summary( alert_title, action_plan, @@ -5002,14 +5125,17 @@ def _build_workflow_alert_content(workflow, run_record, execution_result, priori workflow_name, trigger_source, ) - notification_title = f'{priority.capitalize()} priority workflow alert: {alert_title}' + notification_title = f'{severity_label.capitalize()} priority workflow alert: {alert_title}' else: failure_text = error_text or response_preview or reply_text or ( f'{workflow_name} failed from the {trigger_source} trigger.' ) alert_summary = _summarize_workflow_alert_text(failure_text) alert_detail = _normalize_workflow_alert_text(failure_text) - notification_title = f'{priority.capitalize()} priority workflow alert: {workflow_name} failed' + notification_title = f'{severity_label.capitalize()} priority workflow alert: {workflow_name} failed' + + if trigger_section: + alert_detail = f'{trigger_section}\n\n{alert_detail}' if alert_detail else trigger_section return { 'notification_title': notification_title, @@ -5090,12 +5216,106 @@ def _collect_agent_alert_targets(user_id, conversation_id): return _select_preferred_workflow_alert_targets(alert_targets) -def _create_workflow_priority_alert(workflow, run_record, conversation, execution_result=None): - execution_result = execution_result if isinstance(execution_result, dict) else {} - priority = _normalize_workflow_alert_priority(workflow.get('alert_priority')) - if priority == 'none': +def _record_workflow_alert_decision(workflow, run_record, decision): + """Persist a compact record of the alert decision so a run can explain itself.""" + if not isinstance(run_record, dict): + return + + decision = decision if isinstance(decision, dict) else {} + run_record['alert_decision'] = { + 'should_alert': bool(decision.get('should_alert')), + 'severity': decision.get('severity') or '', + 'category': decision.get('category') or '', + 'delivery': decision.get('delivery') or '', + 'mode': decision.get('mode') or '', + 'summary': summarize_alert_decision(decision), + 'matched_rules': [ + { + 'rule_id': match.get('rule_id'), + 'rule_name': match.get('rule_name'), + 'severity': match.get('severity'), + 'condition_type': match.get('condition_type'), + 'reason': match.get('reason'), + } + for match in decision.get('matched_rules') or [] + ], + } + + try: + _save_workflow_run_record(workflow, run_record) + except Exception as exc: + log_event( + f'[WORKFLOW_RUNNER] Failed to persist workflow alert decision: {exc}', + extra={ + 'workflow_id': str((workflow or {}).get('id') or '').strip(), + 'run_id': str(run_record.get('id') or '').strip(), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + + +def _workflow_alert_rules_need_model_evaluation(alert_config, facts): + """Return True when at least one enabled model evaluated rule could still change the outcome. + + Building the evaluator resolves a model client, so it is only worth doing when + a model evaluated rule outranks every deterministic rule that could match. + """ + alert_config = alert_config if isinstance(alert_config, dict) else {} + if alert_config.get('alert_mode') != 'rules': + return False + + for rule in alert_config.get('alert_rules') or []: + if not isinstance(rule, dict) or not rule.get('enabled', True): + continue + condition = rule.get('condition') if isinstance(rule.get('condition'), dict) else {} + if str(condition.get('type') or '').strip().lower() == 'model_evaluation': + return True + return False + + +def _create_workflow_priority_alert(workflow, run_record, conversation, execution_result=None, settings=None): + execution_result = dict(execution_result) if isinstance(execution_result, dict) else {} + if not execution_result.get('agent_alert_signals'): + execution_result['agent_alert_signals'] = get_workflow_alert_signals() + alert_config = resolve_workflow_alert_config(workflow) + if alert_config.get('alert_mode') == 'off': return None + try: + facts = build_workflow_alert_facts(workflow, run_record, execution_result) + model_evaluator = None + if _workflow_alert_rules_need_model_evaluation(alert_config, facts): + model_evaluator = _build_workflow_alert_model_evaluator(workflow, settings) + decision = evaluate_workflow_alert_rules(workflow, facts, model_evaluator=model_evaluator) + except Exception as exc: + log_event( + f'[WORKFLOW_RUNNER] Failed to evaluate workflow alert rules: {exc}', + extra={ + 'workflow_id': str(workflow.get('id') or '').strip(), + 'user_id': str(workflow.get('user_id') or '').strip(), + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return None + + if not decision.get('should_alert'): + _record_workflow_alert_decision(workflow, run_record, decision) + log_event( + f'[WORKFLOW_RUNNER] {summarize_alert_decision(decision)}', + extra={ + 'workflow_id': str(workflow.get('id') or '').strip(), + 'run_id': str(run_record.get('id') or '').strip(), + 'alert_mode': decision.get('mode'), + }, + level=logging.INFO, + ) + return None + + _record_workflow_alert_decision(workflow, run_record, decision) + priority = normalize_alert_severity(decision.get('severity')) + try: user_id = str(workflow.get('user_id') or '').strip() workflow_id = str(workflow.get('id') or '').strip() @@ -5118,12 +5338,27 @@ def _create_workflow_priority_alert(workflow, run_record, conversation, executio run_record, execution_result, priority, + decision=decision, ) metadata = { 'workflow_id': workflow_id, 'workflow_name': workflow_name, 'priority': priority, + 'category': decision.get('category') or 'alert', + 'delivery': decision.get('delivery') or 'popup', + 'alert_mode': decision.get('mode') or 'rules', + 'matched_rules': [ + { + 'rule_id': match.get('rule_id'), + 'rule_name': match.get('rule_name'), + 'severity': match.get('severity'), + 'condition_type': match.get('condition_type'), + 'reason': match.get('reason'), + } + for match in decision.get('matched_rules') or [] + ], + 'trigger_reason': summarize_alert_decision(decision), 'trigger_source': trigger_source, 'run_id': str(run_record.get('id') or '').strip(), 'runner_type': str(workflow.get('runner_type') or '').strip(), @@ -9304,6 +9539,20 @@ def raise_if_cancelled(): return _merge_workflow_task_execution_results(task_results) +def _get_workflow_conversation_for_alert(run_record): + """Build a minimal conversation stub so alerts can link to the run's conversation.""" + run_record = run_record if isinstance(run_record, dict) else {} + conversation_id = str(run_record.get('conversation_id') or '').strip() + if not conversation_id: + return None + + return { + 'id': conversation_id, + 'chat_type': 'workflow', + 'group_id': str(run_record.get('group_id') or '').strip(), + } + + def _finalize_cancelled_workflow_run( workflow, run_record, @@ -9368,10 +9617,15 @@ def _finalize_cancelled_workflow_run( workspace_type=workspace_type, group_id=group_id or None, ) + alert_notification = _create_workflow_priority_alert( + execution_workflow if isinstance(execution_workflow, dict) else workflow, + run_record, + _get_workflow_conversation_for_alert(run_record), + ) return { 'success': True, 'run': run_record, - 'notification': None, + 'notification': alert_notification, 'workflow_updates': { 'last_run_started_at': started_at, 'last_run_at': completed_at, @@ -9389,6 +9643,20 @@ def _finalize_cancelled_workflow_run( def run_personal_workflow(workflow, trigger_source='manual', user_roles=None, actor_user_id=None, run_id=None): + """Execute a workflow and persist a run record.""" + workflow = workflow if isinstance(workflow, dict) else {} + resolved_run_id = str(run_id or create_workflow_run_id()) + with workflow_alert_signal_scope(workflow, resolved_run_id): + return _run_personal_workflow_impl( + workflow, + trigger_source=trigger_source, + user_roles=user_roles, + actor_user_id=actor_user_id, + run_id=resolved_run_id, + ) + + +def _run_personal_workflow_impl(workflow, trigger_source='manual', user_roles=None, actor_user_id=None, run_id=None): """Execute a workflow and persist a run record.""" workflow = workflow if isinstance(workflow, dict) else {} user_id = str(workflow.get('user_id') or '').strip() @@ -9630,6 +9898,7 @@ def run_personal_workflow(workflow, trigger_source='manual', user_roles=None, ac run_record, conversation, execution_result=execution_result, + settings=settings, ), ) @@ -9736,6 +10005,7 @@ def run_personal_workflow(workflow, trigger_source='manual', user_roles=None, ac execution_workflow, run_record, conversation, + settings=settings, ) return { 'success': False, diff --git a/application/single_app/semantic_kernel_plugins/simplechat_plugin.py b/application/single_app/semantic_kernel_plugins/simplechat_plugin.py index 3dae91201..2677a888f 100644 --- a/application/single_app/semantic_kernel_plugins/simplechat_plugin.py +++ b/application/single_app/semantic_kernel_plugins/simplechat_plugin.py @@ -21,6 +21,7 @@ get_simplechat_enabled_function_names, make_group_inactive_for_current_user, normalize_simplechat_capabilities, + raise_workflow_alert_for_current_user, upload_markdown_document_for_current_user, upload_powerpoint_document_for_current_user, upload_word_document_for_current_user, @@ -258,6 +259,26 @@ def create_personal_workflow( ), ) + # bac-check: ignore - raise_workflow_alert_for_current_user refuses outside an active workflow run. + @plugin_function_logger("SimpleChatPlugin") + @kernel_function(description="Raise an alert signal for the workflow run currently executing. severity can be 'info', 'low', 'medium', 'high', or 'critical'. Only the workflow's own alert rules decide whether the signal becomes a notification. Use signal_name when the workflow defines rules for specific named signals.") + def raise_workflow_alert( + self, + severity: str = "medium", + title: str = "", + reason: str = "", + signal_name: str = "", + ) -> dict: + return self._execute_operation( + "raise_workflow_alert", + lambda: raise_workflow_alert_for_current_user( + severity=severity, + title=title, + reason=reason, + signal_name=signal_name, + ), + ) + # bac-check: ignore - create_group_collaboration_conversation_for_current_user validates current-user group role. @plugin_function_logger("SimpleChatPlugin") @kernel_function(description="Create a new invite-managed group multi-user conversation in a group the current user can access. If group_id is omitted, the active group is used. Add current group members as participants to grant access.") diff --git a/application/single_app/static/js/agent_modal_stepper.js b/application/single_app/static/js/agent_modal_stepper.js index 808d3477c..597958e2e 100644 --- a/application/single_app/static/js/agent_modal_stepper.js +++ b/application/single_app/static/js/agent_modal_stepper.js @@ -100,6 +100,12 @@ const SIMPLECHAT_CAPABILITY_DEFINITIONS = [ key: 'create_personal_collaboration_conversation', label: 'Create personal collaborative conversations', description: 'Allow the agent to create personal collaborative conversations and invite participants.' + }, + { + key: 'raise_workflow_alert', + label: 'Raise workflow alerts', + description: 'Allow the agent to raise an alert signal while a workflow run is executing, so the workflow\'s alert rules can notify the owner.', + defaultEnabled: false } ]; const MSGRAPH_CAPABILITY_DEFINITIONS = [ @@ -3306,7 +3312,7 @@ export class AgentModalStepper { getDefaultSimpleChatCapabilities(actionId = '', actionName = '') { const defaults = {}; SIMPLECHAT_CAPABILITY_DEFINITIONS.forEach(definition => { - defaults[definition.key] = true; + defaults[definition.key] = definition.defaultEnabled !== false; }); const action = (this.availableActions || []).find(candidate => { diff --git a/application/single_app/static/js/notifications.js b/application/single_app/static/js/notifications.js index ad0c00608..818476dd0 100644 --- a/application/single_app/static/js/notifications.js +++ b/application/single_app/static/js/notifications.js @@ -179,26 +179,45 @@ function getWorkflowAlertPriorityBadgeClass(priority) { const normalizedPriority = String(priority || '').trim().toLowerCase(); - if (normalizedPriority === 'high') { + if (normalizedPriority === 'critical' || normalizedPriority === 'high') { return 'text-bg-danger'; } if (normalizedPriority === 'medium') { return 'text-bg-warning'; } - if (normalizedPriority === 'low') { + if (normalizedPriority === 'low' || normalizedPriority === 'info') { return 'text-bg-info'; } return 'text-bg-secondary'; } - function formatWorkflowAlertPriorityLabel(priority) { + function formatWorkflowAlertPriorityLabel(priority, category) { const normalizedPriority = String(priority || '').trim().toLowerCase(); + const normalizedCategory = String(category || '').trim().toLowerCase(); if (!normalizedPriority) { - return 'ALERT'; + return normalizedCategory === 'failure' ? 'FAILURE' : 'ALERT'; + } + if (normalizedCategory === 'failure') { + return `${normalizedPriority.toUpperCase()} FAILURE`; + } + if (normalizedPriority === 'info') { + return 'INFO'; } return `${normalizedPriority.toUpperCase()} PRIORITY`; } + function getWorkflowAlertCategory(notification) { + const metadata = notification?.metadata || {}; + const category = String(metadata.category || notification?.category || 'alert').trim().toLowerCase(); + return category === 'failure' ? 'failure' : 'alert'; + } + + function getWorkflowAlertDelivery(notification) { + const metadata = notification?.metadata || {}; + const delivery = String(metadata.delivery || notification?.delivery || '').trim().toLowerCase(); + return delivery === 'notify_only' ? 'notify_only' : 'popup'; + } + function formatWorkflowTriggeredTime(isoString) { if (!isoString) { return 'Triggered: --:--:--'; @@ -342,7 +361,7 @@ } const normalizedNotificationTitle = normalizeWorkflowAlertText(notification?.title || ''); - const strippedNotificationTitle = normalizedNotificationTitle.replace(/^(low|medium|high)\s+priority\s+workflow\s+alert:\s*/i, '').trim(); + const strippedNotificationTitle = normalizedNotificationTitle.replace(/^(info|low|medium|high|critical)\s+priority\s+workflow\s+alert:\s*/i, '').trim(); if (strippedNotificationTitle) { return strippedNotificationTitle; } @@ -440,10 +459,14 @@ if (workflowAlertModalContent) { workflowAlertModalContent.dataset.priority = priority || 'medium'; + workflowAlertModalContent.dataset.category = getWorkflowAlertCategory(notification); } if (workflowAlertPriorityBadge) { workflowAlertPriorityBadge.className = `badge text-uppercase ${getWorkflowAlertPriorityBadgeClass(priority)}`; - workflowAlertPriorityBadge.textContent = formatWorkflowAlertPriorityLabel(priority); + workflowAlertPriorityBadge.textContent = formatWorkflowAlertPriorityLabel( + priority, + getWorkflowAlertCategory(notification), + ); } if (workflowAlertTitle) { workflowAlertTitle.textContent = alertTitle || workflowName || 'Workflow alert'; @@ -514,6 +537,10 @@ if (queuedIds.has(notification.id) || activeWorkflowAlert?.id === notification.id || hasShownWorkflowAlert(notification.id)) { return; } + // Quiet severities stay in the notification bell instead of interrupting. + if (getWorkflowAlertDelivery(notification) === 'notify_only') { + return; + } queuedIds.add(notification.id); workflowAlertQueue.push(notification); diff --git a/application/single_app/static/js/plugin_modal_stepper.js b/application/single_app/static/js/plugin_modal_stepper.js index 13d934270..4e90e890c 100644 --- a/application/single_app/static/js/plugin_modal_stepper.js +++ b/application/single_app/static/js/plugin_modal_stepper.js @@ -188,6 +188,12 @@ const SIMPLECHAT_CAPABILITY_DEFINITIONS = [ key: 'create_personal_collaboration_conversation', label: 'Create personal collaborative conversations', description: 'Allow this action to create personal collaborative conversations and invite participants.' + }, + { + key: 'raise_workflow_alert', + label: 'Raise workflow alerts', + description: 'Allow this action to raise an alert signal while a workflow run is executing, so the workflow\'s alert rules can notify the owner.', + defaultEnabled: false } ]; const MSGRAPH_CAPABILITY_DEFINITIONS = [ @@ -1309,7 +1315,7 @@ export class PluginModalStepper { getDefaultSimpleChatCapabilities() { const defaults = {}; SIMPLECHAT_CAPABILITY_DEFINITIONS.forEach(definition => { - defaults[definition.key] = true; + defaults[definition.key] = definition.defaultEnabled !== false; }); return defaults; } diff --git a/application/single_app/static/js/workspace/workspace_workflows.js b/application/single_app/static/js/workspace/workspace_workflows.js index 0a3c907b5..437285fa5 100644 --- a/application/single_app/static/js/workspace/workspace_workflows.js +++ b/application/single_app/static/js/workspace/workspace_workflows.js @@ -127,6 +127,12 @@ const workflowFileSyncContinueModeSelect = document.getElementById("workflow-fil const workflowFileSyncUseChangedDocumentsToggle = document.getElementById("workflow-file-sync-use-changed-documents"); const workflowFileSyncHelp = document.getElementById("workflow-file-sync-help"); const workflowAlertPrioritySelect = document.getElementById("workflow-alert-priority"); +const workflowAlertModeSelect = document.getElementById("workflow-alert-mode"); +const workflowAlertPriorityGroup = document.getElementById("workflow-alert-priority-group"); +const workflowAlertRulesGroup = document.getElementById("workflow-alert-rules-group"); +const workflowAlertRulesList = document.getElementById("workflow-alert-rules-list"); +const workflowAlertRuleAddBtn = document.getElementById("workflow-alert-rule-add-btn"); +const workflowAlertEvaluationOnErrorSelect = document.getElementById("workflow-alert-evaluation-on-error"); const workflowErrorStrategyInputs = Array.from(document.querySelectorAll('input[name="workflow-error-strategy"]')); const workflowTaskRetryCountInput = document.getElementById("workflow-task-retry-count"); const workflowReviewSummary = document.getElementById("workflow-review-summary"); @@ -152,6 +158,56 @@ function getWorkflowMaxTasks() { return Math.min(WORKFLOW_TASK_LIMIT_MAX, Math.max(WORKFLOW_TASK_LIMIT_MIN, configuredLimit)); } const WORKFLOW_MAX_TASKS = getWorkflowMaxTasks(); +const WORKFLOW_ALERT_MAX_RULES = 20; +const WORKFLOW_ALERT_SEVERITIES = [ + { value: "info", label: "Info" }, + { value: "low", label: "Low" }, + { value: "medium", label: "Medium" }, + { value: "high", label: "High" }, + { value: "critical", label: "Critical" }, +]; +const WORKFLOW_ALERT_DELIVERIES = [ + { value: "default", label: "Default for severity" }, + { value: "notify_only", label: "Notification bell only" }, + { value: "popup", label: "Pop-up alert" }, +]; +const WORKFLOW_ALERT_CONDITION_TYPES = [ + { value: "run_status", label: "Run finished with a status" }, + { value: "task_status", label: "A task finished with a status" }, + { value: "text_match", label: "Output text matches" }, + { value: "file_sync", label: "File Sync result" }, + { value: "no_output", label: "The run produced no output" }, + { value: "model_evaluation", label: "A model judges a condition" }, + { value: "agent_signal", label: "The agent raised an alert" }, +]; +const WORKFLOW_ALERT_RUN_STATUSES = [ + { value: "failed", label: "Failed" }, + { value: "completed", label: "Completed" }, + { value: "completed_with_task_errors", label: "Completed with task errors" }, + { value: "cancelled", label: "Cancelled" }, +]; +const WORKFLOW_ALERT_TASK_STATUSES = [ + { value: "failed", label: "Failed" }, + { value: "succeeded", label: "Succeeded" }, +]; +const WORKFLOW_ALERT_TEXT_MATCH_MODES = [ + { value: "contains_any", label: "Contains any of" }, + { value: "contains_all", label: "Contains all of" }, + { value: "not_contains", label: "Does not contain" }, + { value: "regex", label: "Matches regex" }, +]; +const WORKFLOW_ALERT_FILE_SYNC_OUTCOMES = [ + { value: "changes_found", label: "Changed documents were found" }, + { value: "no_changes", label: "No changed documents were found" }, + { value: "sync_failed", label: "File Sync failed" }, +]; +const WORKFLOW_ALERT_SCOPE_TYPES = [ + { value: "final", label: "Final output" }, + { value: "any_task", label: "Any task output" }, + { value: "task", label: "A specific task" }, +]; +// Conditions that read run-level facts rather than a particular output. +const WORKFLOW_ALERT_SCOPELESS_CONDITIONS = new Set(["run_status", "file_sync", "agent_signal"]); const DOCUMENT_ACTION_NONE = "none"; const DOCUMENT_ACTION_SEARCH = "search"; const DOCUMENT_ACTION_ANALYZE = "analyze"; @@ -339,6 +395,7 @@ let workflowPickerDocumentIds = []; let workflowSavedComparisonTargetIds = []; let workflowSavedComparisonPreferredLeftId = ""; let workflowTasks = []; +let workflowAlertRules = []; let activeWorkflowTaskId = ""; let currentWorkflowStepIndex = 0; @@ -838,6 +895,28 @@ function getWorkflowErrorStrategy() { return normalizeText(workflowErrorStrategyInputs.find((input) => input.checked)?.value) || "halt"; } +function getWorkflowAlertReviewSummary(legacyAlertLabel) { + const mode = getWorkflowAlertMode(); + if (mode === "every_run") { + return `Every run - ${normalizeText(legacyAlertLabel)}`; + } + if (mode !== "rules") { + return "Never notify me"; + } + + const enabledRules = workflowAlertRules.filter((rule) => rule.enabled !== false); + if (!enabledRules.length) { + return "No alert rules yet"; + } + + const ruleNames = enabledRules + .map((rule, index) => normalizeText(rule.name) || `Alert rule ${index + 1}`) + .slice(0, 3) + .join(", "); + const suffix = enabledRules.length > 3 ? `, +${enabledRules.length - 3} more` : ""; + return `${enabledRules.length} ${enabledRules.length === 1 ? "rule" : "rules"}: ${ruleNames}${suffix}`; +} + function addWorkflowReviewItem(labelText, valueText) { if (!workflowReviewSummary) { return; @@ -874,7 +953,7 @@ function renderWorkflowReview() { addWorkflowReviewItem("Workspace documents", normalizeText(documentActionLabel)); addWorkflowReviewItem("File Sync", workflowFileSyncEnabledToggle?.checked ? "Before each run" : "Not used"); addWorkflowReviewItem("Failure handling", `${strategyLabel}; ${retryCount} ${retryCount === 1 ? "retry" : "retries"}`); - addWorkflowReviewItem("Completion alert", normalizeText(alertLabel)); + addWorkflowReviewItem("Alerts", getWorkflowAlertReviewSummary(alertLabel)); workflowTasks.forEach((task, index) => { addWorkflowReviewItem( `Task ${index + 1}`, @@ -965,6 +1044,9 @@ function validateWorkflowStep(stepName) { return false; } } + if (stepName === "review" && !validateWorkflowAlertRules()) { + return false; + } return true; } @@ -1141,12 +1223,500 @@ function getWorkflowTriggerLabel(workflow) { } function getWorkflowAlertLabel(workflow) { + const mode = normalizeText(workflow?.alert_mode).toLowerCase(); + const rules = Array.isArray(workflow?.alert_rules) ? workflow.alert_rules : []; const priority = normalizeText(workflow?.alert_priority).toLowerCase(); + + if (mode === "rules" || (!mode && rules.length)) { + const enabledCount = rules.filter((rule) => rule?.enabled !== false).length; + if (!enabledCount) { + return "Off"; + } + return `${enabledCount} ${enabledCount === 1 ? "rule" : "rules"}`; + } + + if (mode === "every_run" || (!mode && priority && priority !== "none")) { + if (!priority || priority === "none") { + return "Off"; + } + return `Every run (${priority})`; + } + + return "Off"; +} + +function createWorkflowAlertRuleId() { + if (window.crypto?.randomUUID) { + return window.crypto.randomUUID(); + } + return `alert-rule-${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + +function normalizeWorkflowAlertRule(rawRule) { + const rule = rawRule && typeof rawRule === "object" ? rawRule : {}; + const condition = rule.condition && typeof rule.condition === "object" ? rule.condition : {}; + const scope = rule.scope && typeof rule.scope === "object" ? rule.scope : {}; + const conditionType = normalizeText(condition.type).toLowerCase() || "run_status"; + const statuses = Array.isArray(condition.statuses) ? condition.statuses.map(normalizeText).filter(Boolean) : []; + + return { + id: normalizeText(rule.id) || createWorkflowAlertRuleId(), + name: normalizeText(rule.name), + enabled: rule.enabled !== false, + severity: normalizeText(rule.severity).toLowerCase() || "medium", + delivery: normalizeText(rule.delivery).toLowerCase() || "default", + scopeType: normalizeText(scope.type).toLowerCase() || "final", + scopeTaskId: normalizeText(scope.task_id), + conditionType, + status: statuses[0] || (conditionType === "task_status" ? "failed" : "failed"), + matchMode: normalizeText(condition.mode).toLowerCase() || "contains_any", + matchValues: Array.isArray(condition.values) ? joinCsvList(condition.values) : "", + matchPattern: normalizeText(condition.pattern), + caseSensitive: Boolean(condition.case_sensitive), + fileSyncOutcome: normalizeText(condition.outcome).toLowerCase() || "changes_found", + evaluationPrompt: normalizeText(condition.prompt), + signalName: normalizeText(condition.signal_name), + minSeverity: normalizeText(condition.min_severity).toLowerCase() || "info", + }; +} + +function createDefaultWorkflowAlertRule() { + return normalizeWorkflowAlertRule({ + name: "Run failed", + severity: "high", + condition: { type: "run_status", statuses: ["failed"] }, + }); +} + +function createWorkflowAlertSelect(options, selectedValue, onChange) { + const select = document.createElement("select"); + select.className = "form-select form-select-sm"; + options.forEach((option) => { + const optionElement = document.createElement("option"); + optionElement.value = option.value; + optionElement.textContent = option.label; + select.appendChild(optionElement); + }); + select.value = selectedValue; + select.addEventListener("change", (event) => onChange(normalizeText(event.target.value))); + return select; +} + +function createWorkflowAlertInput(value, placeholder, onChange, multiline = false) { + const input = document.createElement(multiline ? "textarea" : "input"); + input.className = "form-control form-control-sm"; + if (multiline) { + input.rows = 2; + } else { + input.type = "text"; + } + input.value = value; + input.placeholder = placeholder; + input.addEventListener("input", (event) => onChange(event.target.value)); + return input; +} + +function createWorkflowAlertField(labelText, controlElement, columnClass = "col-md-4") { + const column = document.createElement("div"); + column.className = columnClass; + const label = document.createElement("label"); + label.className = "form-label small mb-1"; + label.textContent = labelText; + const controlId = `alert-field-${Math.random().toString(16).slice(2)}`; + label.setAttribute("for", controlId); + controlElement.id = controlId; + column.append(label, controlElement); + return column; +} + +function getWorkflowAlertTaskOptions() { + return workflowTasks.map((task, index) => ({ + value: task.id, + label: normalizeText(task.name) || `Task ${index + 1}`, + })); +} + +function buildWorkflowAlertConditionFields(rule) { + const fields = []; + + if (rule.conditionType === "run_status") { + fields.push(createWorkflowAlertField( + "Run status", + createWorkflowAlertSelect(WORKFLOW_ALERT_RUN_STATUSES, rule.status, (value) => { + rule.status = value; + }), + )); + } else if (rule.conditionType === "task_status") { + fields.push(createWorkflowAlertField( + "Task status", + createWorkflowAlertSelect(WORKFLOW_ALERT_TASK_STATUSES, rule.status, (value) => { + rule.status = value; + }), + )); + } else if (rule.conditionType === "text_match") { + fields.push(createWorkflowAlertField( + "Match type", + createWorkflowAlertSelect(WORKFLOW_ALERT_TEXT_MATCH_MODES, rule.matchMode, (value) => { + rule.matchMode = value; + renderWorkflowAlertRules(); + }), + )); + if (rule.matchMode === "regex") { + fields.push(createWorkflowAlertField( + "Regex pattern", + createWorkflowAlertInput(rule.matchPattern, "expires in \\d+ days", (value) => { + rule.matchPattern = value; + }), + "col-md-5", + )); + } else { + fields.push(createWorkflowAlertField( + "Text values (comma separated)", + createWorkflowAlertInput(rule.matchValues, "EXPIRING, CRITICAL", (value) => { + rule.matchValues = value; + }), + "col-md-5", + )); + } + } else if (rule.conditionType === "file_sync") { + fields.push(createWorkflowAlertField( + "File Sync result", + createWorkflowAlertSelect(WORKFLOW_ALERT_FILE_SYNC_OUTCOMES, rule.fileSyncOutcome, (value) => { + rule.fileSyncOutcome = value; + }), + "col-md-5", + )); + } else if (rule.conditionType === "model_evaluation") { + fields.push(createWorkflowAlertField( + "Condition to judge", + createWorkflowAlertInput( + rule.evaluationPrompt, + "any certificate expires within 14 days", + (value) => { + rule.evaluationPrompt = value; + }, + true, + ), + "col-md-8", + )); + } else if (rule.conditionType === "agent_signal") { + fields.push(createWorkflowAlertField( + "Signal name (optional)", + createWorkflowAlertInput(rule.signalName, "expiring-certificates", (value) => { + rule.signalName = value; + }), + )); + fields.push(createWorkflowAlertField( + "Minimum signal severity", + createWorkflowAlertSelect(WORKFLOW_ALERT_SEVERITIES, rule.minSeverity, (value) => { + rule.minSeverity = value; + }), + )); + } + + return fields; +} + +function buildWorkflowAlertRuleRow(rule, index) { + const card = document.createElement("div"); + card.className = "card p-2 mb-2 workflow-alert-rule"; + card.dataset.ruleId = rule.id; + + const primaryRow = document.createElement("div"); + primaryRow.className = "row g-2"; + primaryRow.append( + createWorkflowAlertField( + "Rule name", + createWorkflowAlertInput(rule.name, `Alert rule ${index + 1}`, (value) => { + rule.name = value; + }), + ), + createWorkflowAlertField( + "Condition", + createWorkflowAlertSelect(WORKFLOW_ALERT_CONDITION_TYPES, rule.conditionType, (value) => { + rule.conditionType = value; + renderWorkflowAlertRules(); + }), + ), + createWorkflowAlertField( + "Severity", + createWorkflowAlertSelect(WORKFLOW_ALERT_SEVERITIES, rule.severity, (value) => { + rule.severity = value; + }), + "col-md-2", + ), + createWorkflowAlertField( + "Delivery", + createWorkflowAlertSelect(WORKFLOW_ALERT_DELIVERIES, rule.delivery, (value) => { + rule.delivery = value; + }), + "col-md-2", + ), + ); + card.appendChild(primaryRow); + + const conditionFields = buildWorkflowAlertConditionFields(rule); + if (!WORKFLOW_ALERT_SCOPELESS_CONDITIONS.has(rule.conditionType)) { + conditionFields.push(createWorkflowAlertField( + "Look at", + createWorkflowAlertSelect(WORKFLOW_ALERT_SCOPE_TYPES, rule.scopeType, (value) => { + rule.scopeType = value; + renderWorkflowAlertRules(); + }), + "col-md-3", + )); + if (rule.scopeType === "task") { + const taskOptions = getWorkflowAlertTaskOptions(); + conditionFields.push(createWorkflowAlertField( + "Task", + createWorkflowAlertSelect( + taskOptions.length ? taskOptions : [{ value: "", label: "Add a task first" }], + rule.scopeTaskId, + (value) => { + rule.scopeTaskId = value; + }, + ), + "col-md-3", + )); + } + } + + if (conditionFields.length) { + const conditionRow = document.createElement("div"); + conditionRow.className = "row g-2 mt-1"; + conditionFields.forEach((field) => conditionRow.appendChild(field)); + card.appendChild(conditionRow); + } + + if (rule.conditionType === "text_match" && rule.matchMode !== "regex") { + const caseWrapper = document.createElement("div"); + caseWrapper.className = "form-check mt-2"; + const caseInput = document.createElement("input"); + caseInput.className = "form-check-input"; + caseInput.type = "checkbox"; + caseInput.id = `alert-case-${rule.id}`; + caseInput.checked = Boolean(rule.caseSensitive); + caseInput.addEventListener("change", (event) => { + rule.caseSensitive = Boolean(event.target.checked); + }); + const caseLabel = document.createElement("label"); + caseLabel.className = "form-check-label small"; + caseLabel.setAttribute("for", caseInput.id); + caseLabel.textContent = "Match upper and lower case exactly"; + caseWrapper.append(caseInput, caseLabel); + card.appendChild(caseWrapper); + } + + const footer = document.createElement("div"); + footer.className = "d-flex justify-content-between align-items-center mt-2"; + + const enabledWrapper = document.createElement("div"); + enabledWrapper.className = "form-check form-switch"; + const enabledInput = document.createElement("input"); + enabledInput.className = "form-check-input"; + enabledInput.type = "checkbox"; + enabledInput.id = `alert-enabled-${rule.id}`; + enabledInput.checked = rule.enabled !== false; + enabledInput.addEventListener("change", (event) => { + rule.enabled = Boolean(event.target.checked); + }); + const enabledLabel = document.createElement("label"); + enabledLabel.className = "form-check-label small ms-2"; + enabledLabel.setAttribute("for", enabledInput.id); + enabledLabel.textContent = "Rule enabled"; + enabledWrapper.append(enabledInput, enabledLabel); + + const removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "btn btn-sm btn-outline-danger"; + removeButton.textContent = "Remove"; + removeButton.addEventListener("click", () => { + workflowAlertRules = workflowAlertRules.filter((item) => item.id !== rule.id); + renderWorkflowAlertRules(); + renderWorkflowReview(); + }); + + footer.append(enabledWrapper, removeButton); + card.appendChild(footer); + return card; +} + +function renderWorkflowAlertRules() { + if (!workflowAlertRulesList) { + return; + } + + clearElementChildren(workflowAlertRulesList); + if (!workflowAlertRules.length) { + const emptyState = document.createElement("div"); + emptyState.className = "text-muted small"; + emptyState.textContent = "No alert rules yet. Add a rule to choose what should notify you."; + workflowAlertRulesList.appendChild(emptyState); + return; + } + + workflowAlertRules.forEach((rule, index) => { + workflowAlertRulesList.appendChild(buildWorkflowAlertRuleRow(rule, index)); + }); +} + +function getWorkflowAlertMode() { + return normalizeText(workflowAlertModeSelect?.value).toLowerCase() || "off"; +} + +function updateWorkflowAlertModeVisibility() { + const mode = getWorkflowAlertMode(); + setElementVisibility(workflowAlertPriorityGroup, mode === "every_run"); + setElementVisibility(workflowAlertRulesGroup, mode === "rules"); +} + +function buildWorkflowAlertRulePayload(rule) { + const condition = { type: rule.conditionType }; + if (rule.conditionType === "run_status" || rule.conditionType === "task_status") { + condition.statuses = [rule.status]; + } else if (rule.conditionType === "text_match") { + condition.mode = rule.matchMode; + if (rule.matchMode === "regex") { + condition.pattern = normalizeText(rule.matchPattern); + } else { + condition.values = parseCsvList(rule.matchValues); + condition.case_sensitive = Boolean(rule.caseSensitive); + } + } else if (rule.conditionType === "file_sync") { + condition.outcome = rule.fileSyncOutcome; + } else if (rule.conditionType === "model_evaluation") { + condition.prompt = normalizeText(rule.evaluationPrompt); + } else if (rule.conditionType === "agent_signal") { + condition.signal_name = normalizeText(rule.signalName); + condition.min_severity = rule.minSeverity; + } + + const scopeType = WORKFLOW_ALERT_SCOPELESS_CONDITIONS.has(rule.conditionType) ? "final" : rule.scopeType; + return { + id: rule.id, + name: normalizeText(rule.name), + enabled: rule.enabled !== false, + severity: rule.severity, + delivery: rule.delivery, + scope: { + type: scopeType, + task_id: scopeType === "task" ? normalizeText(rule.scopeTaskId) : "", + }, + condition, + }; +} + +function collectWorkflowAlertRulesPayload() { + return workflowAlertRules.map((rule) => buildWorkflowAlertRulePayload(rule)); +} + +function buildLegacyWorkflowAlertRules(priority) { if (!priority || priority === "none") { - return "Off"; + return []; } - return `${priority.charAt(0).toUpperCase()}${priority.slice(1)} priority`; + // Mirrors build_legacy_alert_rules in functions_workflow_alerts.py so an + // unmigrated workflow shows the same rules the runner would apply. + return [ + { + id: "legacy-run-failed", + name: "Run failed", + enabled: true, + severity: "high", + delivery: "popup", + scope: { type: "final", task_id: "" }, + condition: { type: "run_status", statuses: ["failed"] }, + }, + { + id: "legacy-run-completed", + name: "Run completed", + enabled: true, + severity: priority, + delivery: "popup", + scope: { type: "final", task_id: "" }, + condition: { type: "run_status", statuses: ["completed"] }, + }, + ]; +} + +function loadWorkflowAlertSettings(workflow) { + const source = workflow && typeof workflow === "object" ? workflow : {}; + const storedRules = Array.isArray(source.alert_rules) ? source.alert_rules : []; + const storedMode = normalizeText(source.alert_mode).toLowerCase(); + const priority = normalizeText(source.alert_priority).toLowerCase() || "none"; + + let mode = storedMode; + let rules = storedRules; + if (!mode) { + if (storedRules.length) { + mode = "rules"; + } else if (priority !== "none") { + mode = "rules"; + rules = buildLegacyWorkflowAlertRules(priority); + } else { + mode = "off"; + } + } + + workflowAlertRules = rules.map((rule) => normalizeWorkflowAlertRule(rule)); + if (workflowAlertModeSelect) { + workflowAlertModeSelect.value = mode; + } + + const evaluation = source.alert_evaluation && typeof source.alert_evaluation === "object" + ? source.alert_evaluation + : {}; + if (workflowAlertEvaluationOnErrorSelect) { + workflowAlertEvaluationOnErrorSelect.value = normalizeText(evaluation.on_error).toLowerCase() || "skip"; + } + + renderWorkflowAlertRules(); + updateWorkflowAlertModeVisibility(); +} + +function validateWorkflowAlertRules() { + if (getWorkflowAlertMode() !== "rules") { + return true; + } + + if (!workflowAlertRules.length) { + showToast("Add at least one alert rule or choose a different alert mode.", "warning"); + return false; + } + + if (workflowAlertRules.length > WORKFLOW_ALERT_MAX_RULES) { + showToast(`Workflows support up to ${WORKFLOW_ALERT_MAX_RULES} alert rules.`, "warning"); + return false; + } + + for (let index = 0; index < workflowAlertRules.length; index += 1) { + const rule = workflowAlertRules[index]; + const ruleLabel = normalizeText(rule.name) || `Alert rule ${index + 1}`; + + if (rule.conditionType === "text_match") { + if (rule.matchMode === "regex" && !normalizeText(rule.matchPattern)) { + showToast(`Add a regex pattern for "${ruleLabel}".`, "warning"); + return false; + } + if (rule.matchMode !== "regex" && !parseCsvList(rule.matchValues).length) { + showToast(`Add at least one text value for "${ruleLabel}".`, "warning"); + return false; + } + } + + if (rule.conditionType === "model_evaluation" && !normalizeText(rule.evaluationPrompt)) { + showToast(`Describe the condition a model should judge for "${ruleLabel}".`, "warning"); + return false; + } + + const usesScope = !WORKFLOW_ALERT_SCOPELESS_CONDITIONS.has(rule.conditionType); + if (usesScope && rule.scopeType === "task" && !normalizeText(rule.scopeTaskId)) { + showToast(`Select the task that "${ruleLabel}" should watch.`, "warning"); + return false; + } + } + + return true; } function parseCsvList(value) { @@ -3027,6 +3597,15 @@ function resetWorkflowForm() { if (workflowAlertPrioritySelect) { workflowAlertPrioritySelect.value = "none"; } + if (workflowAlertModeSelect) { + workflowAlertModeSelect.value = "off"; + } + if (workflowAlertEvaluationOnErrorSelect) { + workflowAlertEvaluationOnErrorSelect.value = "skip"; + } + workflowAlertRules = []; + renderWorkflowAlertRules(); + updateWorkflowAlertModeVisibility(); workflowErrorStrategyInputs.forEach((input) => { input.checked = input.value === "halt"; }); @@ -3141,6 +3720,7 @@ async function openWorkflowModal(workflow = null) { if (workflowAlertPrioritySelect) { workflowAlertPrioritySelect.value = normalizeText(workflow.alert_priority).toLowerCase() || "none"; } + loadWorkflowAlertSettings(workflow); const errorHandling = workflow.error_handling && typeof workflow.error_handling === "object" ? workflow.error_handling : {}; @@ -3304,6 +3884,11 @@ function buildWorkflowPayload() { : true, trigger_type: triggerType, alert_priority: normalizeText(workflowAlertPrioritySelect?.value).toLowerCase() || "none", + alert_mode: getWorkflowAlertMode(), + alert_rules: collectWorkflowAlertRulesPayload(), + alert_evaluation: { + on_error: normalizeText(workflowAlertEvaluationOnErrorSelect?.value).toLowerCase() || "skip", + }, is_enabled: ["interval", "file_sync"].includes(triggerType) ? Boolean(workflowEnabledToggle?.checked) : true, schedule: {}, file_sync: { @@ -3547,6 +4132,10 @@ async function saveWorkflow(event) { return; } + if (!validateWorkflowAlertRules()) { + return; + } + let payload; try { payload = buildWorkflowPayload(); @@ -4100,6 +4689,23 @@ function initializeWorkflowEvents() { renderWorkflowTasks(); }); workflowAlertPrioritySelect?.addEventListener("change", renderWorkflowReview); + workflowAlertModeSelect?.addEventListener("change", () => { + updateWorkflowAlertModeVisibility(); + if (getWorkflowAlertMode() === "rules" && !workflowAlertRules.length) { + workflowAlertRules.push(createDefaultWorkflowAlertRule()); + renderWorkflowAlertRules(); + } + renderWorkflowReview(); + }); + workflowAlertRuleAddBtn?.addEventListener("click", () => { + if (workflowAlertRules.length >= WORKFLOW_ALERT_MAX_RULES) { + showToast(`Workflows support up to ${WORKFLOW_ALERT_MAX_RULES} alert rules.`, "warning"); + return; + } + workflowAlertRules.push(createDefaultWorkflowAlertRule()); + renderWorkflowAlertRules(); + renderWorkflowReview(); + }); workflowTriggerTypeSelect?.addEventListener("change", updateTriggerFields); workflowScheduleUnitSelect?.addEventListener("change", updateScheduleConstraints); workflowFileSyncEnabledToggle?.addEventListener("change", updateFileSyncFields); diff --git a/application/single_app/templates/base.html b/application/single_app/templates/base.html index 7354a5d21..f5899222a 100644 --- a/application/single_app/templates/base.html +++ b/application/single_app/templates/base.html @@ -332,6 +332,15 @@ background: var(--workflow-alert-accent, var(--bs-primary)); } + .workflow-alert-modal-content[data-priority="critical"] { + --workflow-alert-accent: var(--bs-danger); + --workflow-alert-accent-rgb: var(--bs-danger-rgb); + } + + .workflow-alert-modal-content[data-priority="critical"]::before { + width: 10px; + } + .workflow-alert-modal-content[data-priority="high"] { --workflow-alert-accent: var(--bs-danger); --workflow-alert-accent-rgb: var(--bs-danger-rgb); @@ -342,11 +351,17 @@ --workflow-alert-accent-rgb: var(--bs-warning-rgb); } - .workflow-alert-modal-content[data-priority="low"] { + .workflow-alert-modal-content[data-priority="low"], + .workflow-alert-modal-content[data-priority="info"] { --workflow-alert-accent: var(--bs-info); --workflow-alert-accent-rgb: var(--bs-info-rgb); } + .workflow-alert-modal-content[data-category="failure"] { + --workflow-alert-accent: var(--bs-danger); + --workflow-alert-accent-rgb: var(--bs-danger-rgb); + } + .workflow-alert-modal-header { display: flex; justify-content: space-between; diff --git a/application/single_app/templates/group_workspaces.html b/application/single_app/templates/group_workspaces.html index 017374ca0..40d8ad40b 100644 --- a/application/single_app/templates/group_workspaces.html +++ b/application/single_app/templates/group_workspaces.html @@ -2379,8 +2379,18 @@
Workspace documents
-
+
Alerts
+
Alerts notify you when a run meets a condition you define. A run that matches nothing stays silent.
+
+ + +
+
-
-
When enabled, workflow runs create a pop-up alert modal with direct links to the workflow conversation and any conversation the agent created during the run.
+
+
+
+ Alert rules + +
+
+
When several rules match the same run, the highest severity wins and every matched rule is listed in the alert.
+
Info and low severities go to the notification bell only. Medium and above open the pop-up alert.
+
+
+ + +
diff --git a/application/single_app/templates/workspace.html b/application/single_app/templates/workspace.html index e4f72124e..129b91d4b 100644 --- a/application/single_app/templates/workspace.html +++ b/application/single_app/templates/workspace.html @@ -1775,8 +1775,18 @@
Workspace documents
-
+
Alerts
+
Alerts notify you when a run meets a condition you define. A run that matches nothing stays silent.
+
+ + +
+
-
-
When enabled, workflow runs create a pop-up alert modal with direct links to the workflow conversation and any conversation the agent created during the run.
+
+
+
+ Alert rules + +
+
+
When several rules match the same run, the highest severity wins and every matched rule is listed in the alert.
+
Info and low severities go to the notification bell only. Medium and above open the pop-up alert.
+
+
+ + +
diff --git a/docs/explanation/features/WORKFLOW_ALERT_RULES.md b/docs/explanation/features/WORKFLOW_ALERT_RULES.md new file mode 100644 index 000000000..51900cd13 --- /dev/null +++ b/docs/explanation/features/WORKFLOW_ALERT_RULES.md @@ -0,0 +1,263 @@ +# Workflow Alert Rules + +Conditional, severity-graded notifications for personal and group workflows. + +- **Implemented in version:** `0.250.213` +- **Applies to:** personal workflows and group workflows +- **Dependencies:** existing workflow runner, notification center, Semantic Kernel SimpleChat plugin + +## Overview + +Before this feature a workflow carried a single `alert_priority` field with the values +`none`, `low`, `medium` or `high`. If it was anything other than `none`, the runner created a +notification on **every** run, whether the run succeeded or failed and regardless of what it found. +The alert therefore only told the owner "the workflow ran," which made it easy to either drown in +notifications or turn them off entirely. + +Workflow alert rules replace that single switch with a small rule engine. An owner declares **why** +a workflow should notify them and **how loudly**, and a run that matches nothing produces no +notification at all. + +## Architecture + +``` +run finishes (completed / failed / cancelled) + │ + ▼ +build_workflow_alert_facts() ──► run status, task results, final output, task outputs, + File Sync result, agent raised signals, error text + │ + ▼ +evaluate_workflow_alert_rules() + │ deterministic rules evaluated first (free) + │ model evaluated rules batched into one call, and skipped entirely when a + │ deterministic rule already matched at or above their severity + ▼ +decision { should_alert, severity, category, delivery, matched_rules[], reasons[] } + │ + ├── should_alert = False ──► no notification, decision recorded on the run + └── should_alert = True ──► create_workflow_priority_notification() +``` + +### File structure + +| File | Role | +|---|---| +| `application/single_app/functions_workflow_alerts.py` | Rule schema, validation, evaluators, batched model evaluation, legacy migration | +| `application/single_app/functions_workflow_runner.py` | Builds the facts, calls the engine, records the decision, creates the notification, hosts the agent signal scope | +| `application/single_app/functions_personal_workflows.py` | Normalizes alert settings on personal workflow save | +| `application/single_app/functions_group_workflows.py` | Normalizes alert settings on group workflow save | +| `application/single_app/functions_notifications.py` | Severity and category presentation, notify-only filtering | +| `application/single_app/functions_workflow_activity.py` | Surfaces the configured rules and each run's alert decision | +| `application/single_app/semantic_kernel_plugins/simplechat_plugin.py` | `raise_workflow_alert` kernel function | +| `application/single_app/static/js/workspace/workspace_workflows.js` | Rules editor used by both the personal and group workflow modals | +| `application/single_app/static/js/notifications.js` | Badge styling, failure wording, delivery gating | + +`functions_workflow_alerts.py` deliberately imports neither the workflow runner nor the workflow +CRUD modules, so it can be shared by every caller without an import cycle. The model client is +injected into the engine as a callable for the same reason. + +## Configuration + +Alert settings live on the workflow document: + +```jsonc +{ + "alert_mode": "off | every_run | rules", + "alert_priority": "none | low | medium | high", // only used by every_run + "alert_rules": [ + { + "id": "uuid", + "name": "Expiring certificates", + "enabled": true, + "severity": "info | low | medium | high | critical", + "delivery": "default | notify_only | popup", + "scope": { "type": "final | any_task | task", "task_id": "" }, + "condition": { "type": "...", /* type specific fields */ } + } + ], + "alert_evaluation": { "on_error": "skip | alert" } +} +``` + +### Alert modes + +| Mode | Behavior | +|---|---| +| `off` | The workflow never notifies. | +| `every_run` | The legacy behavior. Every completed or failed run notifies at `alert_priority`. Cancelled runs stay silent, as before. | +| `rules` | Only runs matching an enabled rule notify. | + +Selecting `rules` with an empty rule list, or `every_run` with a priority of `none`, is rejected at +save time so the configuration cannot silently mean "never notify." + +### Condition types + +| Type | Fields | Matches when | +|---|---|---| +| `run_status` | `statuses[]` from `completed`, `failed`, `cancelled`, `completed_with_task_errors` | The run ended in one of the listed statuses | +| `task_status` | `statuses[]` from `succeeded`, `failed` | A task in scope ended in one of the listed statuses | +| `text_match` | `mode` (`contains_any`, `contains_all`, `not_contains`, `regex`), `values[]` or `pattern`, `case_sensitive` | The scoped output matches | +| `file_sync` | `outcome` (`changes_found`, `no_changes`, `sync_failed`) | The pre-run File Sync produced that outcome | +| `no_output` | — | The scoped output was empty | +| `model_evaluation` | `prompt` | A model judges the plain-English condition to be met | +| `agent_signal` | `signal_name`, `min_severity` | The agent raised a matching signal during the run | + +`run_status`, `file_sync` and `agent_signal` read run-level facts, so they ignore scope. Every other +condition honors the rule's scope. + +### Scope + +| Scope | Reads | +|---|---| +| `final` | The final workflow reply | +| `any_task` | Each task's output, evaluated independently | +| `task` | One specific task's output, selected by `task_id` | + +`contains_all` is evaluated per candidate, so `any_task` means "one task output contains all of the +values" rather than "the values appear somewhere across the run." + +### Severity, category and delivery + +| Severity | Icon | Color | Default delivery | +|---|---|---|---| +| `info` | `bi-info-circle` | info | Notification bell only | +| `low` | `bi-bell` | info | Notification bell only | +| `medium` | `bi-exclamation-circle` | warning | Pop-up alert | +| `high` | `bi-exclamation-triangle` | danger | Pop-up alert | +| `critical` | `bi-exclamation-octagon` | danger, wider accent | Pop-up alert | + +Each rule can override the delivery with `notify_only` or `popup`. + +Separately from severity, every alert carries a **category**. When the winning rule was triggered by +a run error, a task failure, a File Sync failure, or an empty run, the category is `failure`, which +swaps the icon to `bi-x-octagon`, forces the danger accent, and changes the badge from +`HIGH PRIORITY` to `HIGH FAILURE`. This keeps "the workflow broke" visually distinct from "the +workflow found something," at whatever severity the owner chose. + +### Resolving multiple matches + +All enabled rules are evaluated. The **highest severity match wins** and sets the severity, category +and delivery; ties are broken by rule order. Every matched rule is still reported, and the alert +detail opens with a `Triggered by` section listing each matched rule, its severity and its reason. + +One run produces at most one notification no matter how many rules match. + +## Model evaluated conditions + +A `model_evaluation` rule holds a plain-English condition such as *"any certificate expires within +14 days."* These are the most useful conditions and the only ones that cost tokens, so they are +tightly bounded: + +- **One call per run at most.** Every pending model evaluated rule is batched into a single prompt + and the model returns one JSON verdict per rule. +- **Skipped when moot.** If a deterministic rule already matched at or above a model rule's + severity, that model rule cannot change the outcome and is not sent. When no model rule remains, + no model client is even resolved. +- **Bounded input.** The scoped output is truncated before it enters the prompt. +- **Strict output.** The response is parsed as JSON, tolerating markdown fences and surrounding + chatter. `alert_evaluation.on_error` decides what happens when it cannot be parsed or the call + fails: `skip` stays silent, `alert` raises the rule as a `failure` so the condition is not + silently dropped. + +The evaluator uses the workflow's own runner model and falls back to the app default. + +## Agent raised alerts + +Agent-runner workflows can raise an alert mid-run through the SimpleChat plugin: + +```text +raise_workflow_alert(severity, title, reason, signal_name) +``` + +The signal is written into a run-scoped `contextvar` that the runner opens around the whole run and +resets in a `finally` block, so signals never leak between runs. Outside an active workflow run the +function refuses with a `PermissionError`, so a normal chat cannot fabricate a workflow +notification. + +An `agent_signal` rule then decides whether the signal notifies anyone. The rule's severity acts as +a **floor**: an agent can escalate above it but never quiet it below. Use `signal_name` to route +distinct signals to distinct rules. + +`raise_workflow_alert` is **opt-in**. It is registered with `default_enabled: False`, so existing +agents do not silently gain the ability to create notifications; an admin enables it per action in +the agent or plugin capability toggles. + +## Usage + +### Configuring rules + +1. Open a workflow in the personal or group workspace and go to the **Review** step. +2. Under **Alerts**, set **When to alert**: + - *Never notify me* for silence. + - *Only when a condition is met* to use rules. + - *On every run* to keep the legacy behavior. +3. In rules mode, add rules with **Add rule**. For each rule choose a name, condition, severity and + delivery, then fill in the condition fields and, where relevant, what the rule should look at. +4. Choose what should happen when a model evaluated condition cannot be judged. + +### Example: alert only on something worth knowing + +| Rule | Condition | Severity | +|---|---|---| +| Run failed | Run status is `failed` | `high` | +| Expiring certificates | A model judges "any certificate expires within 14 days" | `critical` | +| Nightly ran | Run status is `completed` | `info` | + +A clean nightly run produces a quiet bell entry. A run that finds an expiring certificate raises a +critical pop-up listing all three matched rules. A broken run raises a high failure alert. + +## Migration + +Workflows saved before this feature only carry `alert_priority`. They are migrated **on read** by +`resolve_workflow_alert_config()`, so no data migration is required and behavior is preserved +exactly: + +| Legacy value | Migrated rules | +|---|---| +| `none` | Mode `off`, no rules | +| `low` / `medium` / `high` | Mode `rules` with `Run failed → high` and `Run completed → ` | + +Because legacy alerts always opened the modal regardless of priority, the migrated rules pin +`delivery` to `popup` rather than inheriting the new severity default. The rules are ordinary +editable rules, so an owner can prune the noisy one. Cancelled runs did not alert before and still +do not. + +Clients that predate alert rules, such as the `create_personal_workflow` plugin function, still send +only `alert_priority`; the save path materializes the same migrated rules for them. + +## Observability + +Each run records a compact `alert_decision` on its run record, including whether it alerted, the +winning severity and category, and every matched rule with its reason. This is surfaced through +`functions_workflow_activity.py` alongside the workflow's configured rules, so an owner can see why +a run did or did not alert. The decision is also written to App Insights through `log_event`. + +## Limitations and safeguards + +- **Regex safety.** Patterns are capped at 200 characters, compiled at save time, and rejected when + they contain a quantified group that itself contains a quantifier, which is the classic + catastrophic backtracking shape. Searched text is truncated before matching. +- **Rule count.** A workflow supports up to 20 alert rules. +- **One notification per run.** Multiple matches raise the severity and add reasons, they do not + multiply notifications. +- **No cooldown yet.** A condition that persists across a short schedule will alert on every cycle. + Per-rule cooldown and digest rollups are planned follow-ups. +- **Model cost.** Model evaluated conditions add at most one small model call per run. Workflows + using only deterministic conditions add no model calls. + +## Related tests + +| Test | Covers | +|---|---| +| `functional_tests/test_workflow_alert_rules.py` | Every condition type, scoping, severity resolution, failure category, validation, legacy migration | +| `functional_tests/test_workflow_alert_model_evaluation.py` | Batching, the skip optimization, response parsing, `on_error` handling, prompt bounds | +| `functional_tests/test_workflow_alert_agent_signal.py` | Signal normalization, severity floor, named routing, plugin gating and opt-in default | +| `functional_tests/test_workflow_priority_alerts.py` | Existing alert content and deep-link behavior | +| `ui_tests/test_workspace_workflow_alert_rules.py` | The rules editor and legacy rule loading | +| `ui_tests/test_workflow_priority_alert_modal.py` | Critical severity, failure styling, notify-only suppression | + +## Related documentation + +- `docs/explanation/features/WORKFLOW_PRIORITY_ALERTS.md` — the original alert modal and deep links +- `docs/explanation/release_notes.md` — release entry for `0.250.213` diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index e060b6a3c..4b50aa653 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,49 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.213)** + +#### New Features + +* **Workflow Alert Rules** + * Workflow alerts are now conditional. Instead of a single Pop-up Alert Priority that notified on every run, a workflow can define rules that describe *why* it should notify you, and a run that matches nothing stays completely silent. + * Conditions cover run status, task status, output text (contains, does not contain, or regex), File Sync results, empty output, an agent-raised signal, and a plain-English condition judged by a model, such as "any certificate expires within 14 days." + * Each rule can be scoped to the final output, any task output, or one specific task. + * Model-judged conditions are batched into a single call per run and skipped entirely when a deterministic rule already matched at a higher severity, so workflows that use only deterministic conditions add no model calls. + * (Ref: `functions_workflow_alerts.py`, `functions_workflow_runner.py`, `functions_personal_workflows.py`, `functions_group_workflows.py`, `workspace_workflows.js`, `WORKFLOW_ALERT_RULES.md`) + +* **Expanded Alert Severities and a Distinct Failure Style** + * The severity ladder grew from low/medium/high to **info, low, medium, high and critical**. + * Info and low alerts land quietly in the notification bell, while medium and above open the pop-up. Any rule can override this. + * Runs that error now carry a separate *failure* category that changes the icon and wording independently of severity, so "the workflow broke" is visually distinct from "the workflow found something." + * When several rules match the same run, the highest severity wins and the alert lists every matched rule with its reason under a new "Triggered by" section. + * (Ref: `functions_notifications.py`, `notifications.js`, `base.html`, workflow alert modal) + +* **Agent-Raised Workflow Alerts** + * Agents running inside a workflow can now raise an alert signal mid-run with severity, title and reason through the new `raise_workflow_alert` SimpleChat capability, and an `agent_signal` rule decides whether it notifies anyone. + * The rule's severity acts as a floor the agent can escalate above but never quiet below, and named signals can route to their own rules. + * The capability is opt-in and refuses outside an active workflow run, so existing agents do not gain the ability to create notifications and a normal chat cannot fabricate one. + * (Ref: `simplechat_plugin.py`, `functions_simplechat_operations.py`, `agent_modal_stepper.js`, `plugin_modal_stepper.js`) + +* **Alert Decision Visibility** + * Each run now records why it did or did not alert, including the winning severity and every matched rule, surfaced through the workflow activity view so noisy or silent workflows can be diagnosed. + * (Ref: `functions_workflow_activity.py`, `functions_workflow_runner.py`) + +#### User Interface Enhancements + +* **Workflow Alert Rules Editor** + * The Review step of the personal and group workflow builders replaces the single Pop-up Alert Priority dropdown with an alert mode selector and a rule editor for adding, editing, enabling and removing alert rules. + * Each rule row exposes its name, condition, severity, delivery and, where relevant, the task or output it should watch, with condition-specific fields appearing as the condition is chosen. + * The workflow list now summarizes alerts as the number of active rules, and the Review summary names the rules that will notify you. + * Invalid rules are caught before saving, such as a missing regex pattern, empty match values, an unwritten model condition, or a task-scoped rule with no task selected. + * (Ref: `workspace.html`, `group_workspaces.html`, `workspace_workflows.js`, workflow builder review step) + +#### Breaking Changes + +* **Workflow Alert Configuration Model** + * The single `alert_priority` field is superseded by `alert_mode`, `alert_rules` and `alert_evaluation`. The old field is retained and still honored. + * **Migration**: None required. Workflows that only carry `alert_priority` are migrated on read into two editable rules, `Run failed → high` and `Run completed → `, which reproduce the previous behavior exactly, including always opening the pop-up and staying silent on cancelled runs. Owners can then prune the noisy rule. + ### **(v0.250.212)** #### New Features diff --git a/functional_tests/test_workflow_alert_agent_signal.py b/functional_tests/test_workflow_alert_agent_signal.py new file mode 100644 index 000000000..891aabd15 --- /dev/null +++ b/functional_tests/test_workflow_alert_agent_signal.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# test_workflow_alert_agent_signal.py +""" +Functional test for agent raised workflow alert signals. +Version: 0.250.213 +Implemented in: 0.250.213 + +This test ensures an agent can raise an alert signal during a workflow run, that +the signal is normalized and matched by agent_signal rules, that a signal can +escalate a rule's severity but never lower it, and that the plugin function is +gated so it cannot fabricate a notification outside an active workflow run. +""" + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_support.versioning import assert_app_version_at_least + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'application' / 'single_app')) + +from functions_workflow_alerts import ( # noqa: E402 + build_workflow_alert_facts, + evaluate_workflow_alert_rules, + normalize_agent_alert_signal, + normalize_alert_rules, +) + + +def read_text(relative_path): + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def build_workflow(rules): + return { + 'id': 'workflow-1', + 'name': 'Threat watch', + 'alert_mode': 'rules', + 'alert_rules': normalize_alert_rules(rules), + } + + +def decide(workflow, signals, run_record=None): + facts = build_workflow_alert_facts( + workflow, + run_record or {'status': 'completed', 'success': True}, + {'reply': 'run output', 'agent_alert_signals': signals}, + ) + return evaluate_workflow_alert_rules(workflow, facts) + + +def test_agent_signal_normalization(): + """Signals are normalized to a supported severity with trimmed text.""" + print("Testing agent signal normalization...") + signal = normalize_agent_alert_signal({ + 'severity': ' HIGH ', + 'title': ' Breach detected ', + 'reason': 'Unrecognized\n sign-in from a new region.', + 'signal_name': ' suspicious-signin ', + }) + assert signal['severity'] == 'high' + assert signal['title'] == 'Breach detected' + assert signal['reason'] == 'Unrecognized sign-in from a new region.' + assert signal['signal_name'] == 'suspicious-signin' + + # An unknown severity falls back to medium rather than raising mid-run. + assert normalize_agent_alert_signal({'severity': 'catastrophic'})['severity'] == 'medium' + assert normalize_agent_alert_signal({})['severity'] == 'medium' + assert normalize_agent_alert_signal(None)['severity'] == 'medium' + print("Agent signal normalization passed.") + + +def test_agent_signal_matches_and_escalates_but_never_lowers(): + """The rule severity is a floor the agent can raise, not a ceiling it can lower.""" + print("Testing signal severity floor...") + workflow = build_workflow([ + {'id': 'r1', 'name': 'Agent raised', 'severity': 'medium', + 'condition': {'type': 'agent_signal', 'min_severity': 'info'}}, + ]) + + decision = decide(workflow, [{'severity': 'critical', 'reason': 'Active intrusion.'}]) + assert decision['should_alert'] is True + assert decision['severity'] == 'critical' + assert decision['reasons'] == ['Active intrusion.'] + + # A quieter signal still alerts at the rule's configured severity. + decision = decide(workflow, [{'severity': 'info', 'reason': 'Minor note.'}]) + assert decision['should_alert'] is True + assert decision['severity'] == 'medium' + print("Signal severity floor passed.") + + +def test_min_severity_filters_noisy_signals(): + """Signals below the rule's minimum severity are ignored.""" + print("Testing min severity filtering...") + workflow = build_workflow([ + {'id': 'r1', 'name': 'Only serious signals', 'severity': 'high', + 'condition': {'type': 'agent_signal', 'min_severity': 'high'}}, + ]) + + assert decide(workflow, [{'severity': 'medium', 'reason': 'meh'}])['should_alert'] is False + assert decide(workflow, [{'severity': 'high', 'reason': 'serious'}])['should_alert'] is True + assert decide(workflow, [])['should_alert'] is False + print("Min severity filtering passed.") + + +def test_named_signals_route_to_their_own_rules(): + """Distinct named signals can drive distinct rules and severities.""" + print("Testing named signal routing...") + workflow = build_workflow([ + {'id': 'certs', 'name': 'Expiring certificates', 'severity': 'medium', + 'condition': {'type': 'agent_signal', 'signal_name': 'expiring-certificates'}}, + {'id': 'breach', 'name': 'Suspected breach', 'severity': 'critical', + 'condition': {'type': 'agent_signal', 'signal_name': 'suspected-breach'}}, + ]) + + decision = decide(workflow, [{'severity': 'low', 'signal_name': 'expiring-certificates', 'reason': 'Cert expires.'}]) + assert decision['should_alert'] is True + assert decision['severity'] == 'medium' + assert decision['winning_rule_name'] == 'Expiring certificates' + + decision = decide(workflow, [ + {'severity': 'low', 'signal_name': 'expiring-certificates', 'reason': 'Cert expires.'}, + {'severity': 'low', 'signal_name': 'suspected-breach', 'reason': 'Odd traffic.'}, + ]) + assert decision['severity'] == 'critical' + assert {match['rule_name'] for match in decision['matched_rules']} == { + 'Expiring certificates', + 'Suspected breach', + } + + decision = decide(workflow, [{'severity': 'critical', 'signal_name': 'unrelated', 'reason': 'x'}]) + assert decision['should_alert'] is False + print("Named signal routing passed.") + + +def test_plugin_function_is_registered_and_gated(): + """raise_workflow_alert is exposed as a capability and refuses outside a run.""" + print("Testing plugin registration and gating...") + assert_app_version_at_least("0.250.213") + + operations_content = read_text("application/single_app/functions_simplechat_operations.py") + plugin_content = read_text("application/single_app/semantic_kernel_plugins/simplechat_plugin.py") + runner_content = read_text("application/single_app/functions_workflow_runner.py") + + assert '"raise_workflow_alert": "raise_workflow_alert",' in operations_content + assert '"key": "raise_workflow_alert",' in operations_content + # The capability is opt-in so existing agents do not silently gain the ability + # to create notifications when this capability is added. + assert '"default_enabled": False,' in operations_content + assert 'definition.get("default_enabled", True)' in operations_content + assert 'def raise_workflow_alert_for_current_user(' in operations_content + assert 'if not is_workflow_alert_signal_scope_active():' in operations_content + assert 'raise PermissionError(' in operations_content + assert 'Workflow alerts can only be raised while a workflow run is executing.' in operations_content + # The runner import must stay lazy because the runner imports this module. + assert 'from functions_workflow_runner import (' in operations_content + + assert 'raise_workflow_alert_for_current_user,' in plugin_content + assert 'def raise_workflow_alert(' in plugin_content + assert '@kernel_function(description="Raise an alert signal for the workflow run currently executing.' in plugin_content + + # The admin capability toggles mirror the backend registry in both stepper modules. + for stepper_path in ( + "application/single_app/static/js/agent_modal_stepper.js", + "application/single_app/static/js/plugin_modal_stepper.js", + ): + stepper_content = read_text(stepper_path) + assert "key: 'raise_workflow_alert'," in stepper_content + assert 'defaultEnabled: false' in stepper_content + assert 'defaults[definition.key] = definition.defaultEnabled !== false;' in stepper_content + + assert "_workflow_alert_signal_context = ContextVar('workflow_alert_signals', default=None)" in runner_content + assert 'def workflow_alert_signal_scope(workflow=None, run_id=None):' in runner_content + assert 'def is_workflow_alert_signal_scope_active():' in runner_content + assert 'def record_workflow_alert_signal(severity, title=\'\', reason=\'\', signal_name=\'\'):' in runner_content + assert 'def get_workflow_alert_signals():' in runner_content + assert 'with workflow_alert_signal_scope(workflow, resolved_run_id):' in runner_content + assert "execution_result['agent_alert_signals'] = get_workflow_alert_signals()" in runner_content + print("Plugin registration and gating passed.") + + +def test_signal_scope_is_per_run(): + """The signal buffer is scoped to one run so signals never leak between runs.""" + print("Testing per-run signal scope...") + runner_content = read_text("application/single_app/functions_workflow_runner.py") + + # The scope is entered in the public entry point and reset in a finally block, + # so a failed run cannot leave signals behind for the next one. + assert 'token = _workflow_alert_signal_context.set(scope)' in runner_content + assert '_workflow_alert_signal_context.reset(token)' in runner_content + assert 'def _run_personal_workflow_impl(' in runner_content + assert 'return _run_personal_workflow_impl(' in runner_content + + # Group workflows delegate to the same entry point, so they get the same scope. + assert 'def run_group_workflow(' in runner_content + assert 'return run_personal_workflow(' in runner_content + print("Per-run signal scope passed.") + + +if __name__ == '__main__': + tests = [ + test_agent_signal_normalization, + test_agent_signal_matches_and_escalates_but_never_lowers, + test_min_severity_filters_noisy_signals, + test_named_signals_route_to_their_own_rules, + test_plugin_function_is_registered_and_gated, + test_signal_scope_is_per_run, + ] + + results = [] + for test in tests: + try: + test() + results.append(True) + except Exception as exc: + print(f"Test failed: {test.__name__}: {exc}") + import traceback + traceback.print_exc() + results.append(False) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_workflow_alert_model_evaluation.py b/functional_tests/test_workflow_alert_model_evaluation.py new file mode 100644 index 000000000..8054e6f06 --- /dev/null +++ b/functional_tests/test_workflow_alert_model_evaluation.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +# test_workflow_alert_model_evaluation.py +""" +Functional test for model evaluated workflow alert conditions. +Version: 0.250.213 +Implemented in: 0.250.213 + +This test ensures plain-English alert conditions are judged in a single batched +model call, that the call is skipped entirely when a deterministic rule already +matched at a higher severity, and that a malformed or failed evaluation honors +the configured on_error behavior instead of silently misfiring. +""" + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_support.versioning import assert_app_version_at_least + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'application' / 'single_app')) + +from functions_workflow_alerts import ( # noqa: E402 + WORKFLOW_ALERT_EVALUATION_TEXT_LIMIT, + build_model_evaluation_prompt, + build_workflow_alert_facts, + evaluate_workflow_alert_rules, + normalize_alert_rules, + parse_model_evaluation_response, +) + + +def read_text(relative_path): + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def build_workflow(rules, evaluation=None): + return { + 'id': 'workflow-1', + 'name': 'Certificate watch', + 'alert_mode': 'rules', + 'alert_rules': normalize_alert_rules(rules), + 'alert_evaluation': evaluation or {'on_error': 'skip'}, + } + + +def decide(workflow, run_record, execution_result=None, model_evaluator=None): + facts = build_workflow_alert_facts(workflow, run_record, execution_result or {}) + return evaluate_workflow_alert_rules(workflow, facts, model_evaluator=model_evaluator) + + +class RecordingEvaluator: + """Stand-in for the runner supplied model callable.""" + + def __init__(self, response): + self.response = response + self.prompts = [] + + def __call__(self, prompt): + self.prompts.append(prompt) + if isinstance(self.response, Exception): + raise self.response + return self.response + + +def test_model_condition_matches_and_reports_its_reason(): + """A matched plain-English condition alerts with the model's own explanation.""" + print("Testing model evaluated match...") + workflow = build_workflow([ + {'id': 'm1', 'name': 'Expiring certificates', 'severity': 'high', + 'condition': {'type': 'model_evaluation', 'prompt': 'any certificate expires within 14 days'}}, + ]) + evaluator = RecordingEvaluator( + '{"results": [{"rule_id": "m1", "matched": true, "reason": "Two certificates expire in 3 days."}]}' + ) + + decision = decide( + workflow, + {'status': 'completed', 'success': True}, + {'reply': 'cert A expires in 3 days, cert B expires in 3 days'}, + model_evaluator=evaluator, + ) + + assert decision['should_alert'] is True + assert decision['severity'] == 'high' + assert decision['reasons'] == ['Two certificates expire in 3 days.'] + assert decision['model_evaluation']['used'] is True + assert len(evaluator.prompts) == 1 + print("Model evaluated match passed.") + + +def test_model_condition_not_matched_stays_silent(): + """A condition the model rejects produces no notification.""" + print("Testing model evaluated non-match...") + workflow = build_workflow([ + {'id': 'm1', 'name': 'Expiring certificates', 'severity': 'high', + 'condition': {'type': 'model_evaluation', 'prompt': 'any certificate expires within 14 days'}}, + ]) + evaluator = RecordingEvaluator( + '{"results": [{"rule_id": "m1", "matched": false, "reason": "Nothing expires soon."}]}' + ) + + decision = decide( + workflow, + {'status': 'completed', 'success': True}, + {'reply': 'all certificates are valid for another year'}, + model_evaluator=evaluator, + ) + assert decision['should_alert'] is False + assert len(evaluator.prompts) == 1 + print("Model evaluated non-match passed.") + + +def test_multiple_model_conditions_use_a_single_batched_call(): + """Every model evaluated rule is judged in one call, not one call per rule.""" + print("Testing batched evaluation...") + workflow = build_workflow([ + {'id': 'm1', 'name': 'Expiring certs', 'severity': 'medium', + 'condition': {'type': 'model_evaluation', 'prompt': 'any certificate expires soon'}}, + {'id': 'm2', 'name': 'Failed logins', 'severity': 'high', + 'condition': {'type': 'model_evaluation', 'prompt': 'repeated failed sign-ins were reported'}}, + {'id': 'm3', 'name': 'Quota warnings', 'severity': 'low', + 'condition': {'type': 'model_evaluation', 'prompt': 'a quota is close to exhaustion'}}, + ]) + evaluator = RecordingEvaluator( + '{"results": [' + '{"rule_id": "m1", "matched": true, "reason": "One cert expires."},' + '{"rule_id": "m2", "matched": true, "reason": "12 failed sign-ins."},' + '{"rule_id": "m3", "matched": false, "reason": "Quota is fine."}' + ']}' + ) + + decision = decide( + workflow, + {'status': 'completed', 'success': True}, + {'reply': 'cert expiring, 12 failed sign-ins observed'}, + model_evaluator=evaluator, + ) + + assert len(evaluator.prompts) == 1 + assert decision['severity'] == 'high' + matched_ids = {match['rule_id'] for match in decision['matched_rules']} + assert matched_ids == {'m1', 'm2'} + assert 'm1' in evaluator.prompts[0] and 'm2' in evaluator.prompts[0] and 'm3' in evaluator.prompts[0] + print("Batched evaluation passed.") + + +def test_model_call_is_skipped_when_it_cannot_change_the_outcome(): + """No tokens are spent when a deterministic rule already outranks every model rule.""" + print("Testing skip optimization...") + workflow = build_workflow([ + {'id': 'd1', 'name': 'Run failed', 'severity': 'critical', + 'condition': {'type': 'run_status', 'statuses': ['failed']}}, + {'id': 'm1', 'name': 'Expiring certs', 'severity': 'high', + 'condition': {'type': 'model_evaluation', 'prompt': 'any certificate expires soon'}}, + ]) + evaluator = RecordingEvaluator('{"results": []}') + + decision = decide( + workflow, + {'status': 'failed', 'success': False, 'error': 'boom'}, + {}, + model_evaluator=evaluator, + ) + + assert decision['should_alert'] is True + assert decision['severity'] == 'critical' + assert evaluator.prompts == [] + assert decision['model_evaluation']['used'] is False + assert decision['model_evaluation']['skipped_rule_ids'] == ['m1'] + + # A model rule that outranks the deterministic match is still evaluated. + louder_workflow = build_workflow([ + {'id': 'd1', 'name': 'Run completed', 'severity': 'low', + 'condition': {'type': 'run_status', 'statuses': ['completed']}}, + {'id': 'm1', 'name': 'Expiring certs', 'severity': 'critical', + 'condition': {'type': 'model_evaluation', 'prompt': 'any certificate expires soon'}}, + ]) + louder_evaluator = RecordingEvaluator( + '{"results": [{"rule_id": "m1", "matched": true, "reason": "Cert expires tomorrow."}]}' + ) + decision = decide( + louder_workflow, + {'status': 'completed', 'success': True}, + {'reply': 'cert expires tomorrow'}, + model_evaluator=louder_evaluator, + ) + assert len(louder_evaluator.prompts) == 1 + assert decision['severity'] == 'critical' + print("Skip optimization passed.") + + +def test_malformed_and_failed_evaluations_honor_on_error(): + """An unusable verdict either stays silent or alerts, exactly as configured.""" + print("Testing on_error handling...") + rules = [ + {'id': 'm1', 'name': 'Expiring certs', 'severity': 'high', + 'condition': {'type': 'model_evaluation', 'prompt': 'any certificate expires soon'}}, + ] + + skip_workflow = build_workflow(rules, evaluation={'on_error': 'skip'}) + decision = decide( + skip_workflow, + {'status': 'completed', 'success': True}, + {'reply': 'x'}, + model_evaluator=RecordingEvaluator('I could not decide.'), + ) + assert decision['should_alert'] is False + assert decision['model_evaluation']['error'] + + alert_workflow = build_workflow(rules, evaluation={'on_error': 'alert'}) + decision = decide( + alert_workflow, + {'status': 'completed', 'success': True}, + {'reply': 'x'}, + model_evaluator=RecordingEvaluator('I could not decide.'), + ) + assert decision['should_alert'] is True + assert decision['severity'] == 'high' + assert decision['category'] == 'failure' + + # A raised exception is treated the same way as an unparseable verdict. + decision = decide( + alert_workflow, + {'status': 'completed', 'success': True}, + {'reply': 'x'}, + model_evaluator=RecordingEvaluator(RuntimeError('model endpoint unavailable')), + ) + assert decision['should_alert'] is True + assert 'model endpoint unavailable' in decision['model_evaluation']['error'] + + # Without an evaluator the rule is reported as unevaluated rather than matched. + decision = decide( + skip_workflow, + {'status': 'completed', 'success': True}, + {'reply': 'x'}, + model_evaluator=None, + ) + assert decision['should_alert'] is False + assert decision['model_evaluation']['error'] == 'No model evaluator was available for this run.' + print("on_error handling passed.") + + +def test_response_parsing_tolerates_common_model_formatting(): + """Fenced or chatty JSON is still parsed, but nonsense is rejected.""" + print("Testing response parsing...") + verdicts = parse_model_evaluation_response( + '```json\n{"results": [{"rule_id": "m1", "matched": true, "reason": "Found it."}]}\n```' + ) + assert verdicts['m1']['matched'] is True + assert verdicts['m1']['reason'] == 'Found it.' + + verdicts = parse_model_evaluation_response( + 'Sure! {"results": [{"rule_id": "m1", "matched": "yes", "reason": "Found it."}]} Hope that helps.' + ) + assert verdicts['m1']['matched'] is True + + verdicts = parse_model_evaluation_response('{"results": [{"rule_id": "m1", "matched": false}]}') + assert verdicts['m1']['matched'] is False + + for bad_response in ['', 'no json here', '{"other": []}', '[]']: + try: + parse_model_evaluation_response(bad_response) + raise AssertionError(f'Expected rejection for {bad_response!r}') + except ValueError: + pass + print("Response parsing passed.") + + +def test_prompt_includes_scoped_output_and_is_truncated(): + """The prompt carries the scoped output but stays bounded in size.""" + print("Testing prompt construction...") + rules = normalize_alert_rules([ + {'id': 'm1', 'name': 'Scan review', 'severity': 'high', 'scope': {'type': 'any_task'}, + 'condition': {'type': 'model_evaluation', 'prompt': 'the scan reported anything unusual'}}, + ]) + workflow = {'id': 'w', 'name': 'Scanner', 'alert_mode': 'rules', 'alert_rules': rules} + facts = build_workflow_alert_facts( + workflow, + {'status': 'completed', 'success': True}, + { + 'reply': 'final summary', + 'task_results': [ + {'task': {'id': 't1', 'name': 'Scan'}, 'status': 'succeeded', + 'result': {'reply': 'unusual traffic spike detected'}}, + ], + }, + ) + + prompt = build_model_evaluation_prompt(rules, facts) + assert 'unusual traffic spike detected' in prompt + assert 'the scan reported anything unusual' in prompt + assert 'rule_id: m1' in prompt + assert '"results"' in prompt + assert 'Run status: completed' in prompt + + long_facts = build_workflow_alert_facts( + workflow, + {'status': 'completed', 'success': True}, + { + 'reply': 'x', + 'task_results': [ + {'task': {'id': 't1', 'name': 'Scan'}, 'status': 'succeeded', + 'result': {'reply': 'y' * (WORKFLOW_ALERT_EVALUATION_TEXT_LIMIT * 3)}}, + ], + }, + ) + long_prompt = build_model_evaluation_prompt(rules, long_facts) + assert len(long_prompt) < WORKFLOW_ALERT_EVALUATION_TEXT_LIMIT + 2000 + print("Prompt construction passed.") + + +def test_runner_supplies_the_model_evaluator(): + """The runner builds the evaluator only when a model evaluated rule is present.""" + print("Testing runner evaluator wiring...") + assert_app_version_at_least("0.250.213") + runner_content = read_text("application/single_app/functions_workflow_runner.py") + + assert 'def _build_workflow_alert_model_evaluator(workflow, settings=None):' in runner_content + assert '_resolve_model_workflow_client(workflow, evaluation_settings)' in runner_content + assert 'if _workflow_alert_rules_need_model_evaluation(alert_config, facts):' in runner_content + assert 'model_evaluator=model_evaluator' in runner_content + assert "'role': 'system'," in runner_content + assert 'temperature=0,' in runner_content + + workflow_js = read_text("application/single_app/static/js/workspace/workspace_workflows.js") + assert 'model_evaluation' in workflow_js + assert 'workflowAlertEvaluationOnErrorSelect' in workflow_js + print("Runner evaluator wiring passed.") + + +if __name__ == '__main__': + tests = [ + test_model_condition_matches_and_reports_its_reason, + test_model_condition_not_matched_stays_silent, + test_multiple_model_conditions_use_a_single_batched_call, + test_model_call_is_skipped_when_it_cannot_change_the_outcome, + test_malformed_and_failed_evaluations_honor_on_error, + test_response_parsing_tolerates_common_model_formatting, + test_prompt_includes_scoped_output_and_is_truncated, + test_runner_supplies_the_model_evaluator, + ] + + results = [] + for test in tests: + try: + test() + results.append(True) + except Exception as exc: + print(f"Test failed: {test.__name__}: {exc}") + import traceback + traceback.print_exc() + results.append(False) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_workflow_alert_rules.py b/functional_tests/test_workflow_alert_rules.py new file mode 100644 index 000000000..83880c22b --- /dev/null +++ b/functional_tests/test_workflow_alert_rules.py @@ -0,0 +1,577 @@ +#!/usr/bin/env python3 +# test_workflow_alert_rules.py +""" +Functional test for conditional workflow alert rules. +Version: 0.250.213 +Implemented in: 0.250.213 + +This test ensures workflow alerts only fire when a declared condition is met, +that the highest matching severity wins while every matched rule is reported, +that a run matching nothing produces no notification at all, and that legacy +workflows carrying only alert_priority keep alerting through migrated rules. +""" + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_support.versioning import assert_app_version_at_least + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'application' / 'single_app')) + +from functions_workflow_alerts import ( # noqa: E402 + WORKFLOW_ALERT_SEVERITY_ORDER, + build_legacy_alert_rules, + build_workflow_alert_facts, + evaluate_workflow_alert_rules, + get_alert_severity_rank, + normalize_alert_rules, + normalize_workflow_alert_settings, + resolve_workflow_alert_config, + summarize_alert_decision, + validate_alert_regex, +) + + +def read_text(relative_path): + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def build_workflow(rules, mode='rules', evaluation=None): + return { + 'id': 'workflow-1', + 'name': 'Certificate watch', + 'user_id': 'user-1', + 'alert_mode': mode, + 'alert_rules': normalize_alert_rules(rules), + 'alert_evaluation': evaluation or {'on_error': 'skip'}, + } + + +def decide(workflow, run_record, execution_result=None, model_evaluator=None): + facts = build_workflow_alert_facts(workflow, run_record, execution_result or {}) + return evaluate_workflow_alert_rules(workflow, facts, model_evaluator=model_evaluator) + + +def test_severity_ladder_and_delivery_defaults(): + """Info and low stay in the bell; medium and above interrupt with the pop-up.""" + print("Testing severity ladder and delivery defaults...") + assert WORKFLOW_ALERT_SEVERITY_ORDER == ('info', 'low', 'medium', 'high', 'critical') + assert get_alert_severity_rank('critical') > get_alert_severity_rank('high') + assert get_alert_severity_rank('info') < get_alert_severity_rank('low') + + quiet = build_workflow([ + {'id': 'r1', 'name': 'Quiet', 'severity': 'info', + 'condition': {'type': 'run_status', 'statuses': ['completed']}}, + ]) + decision = decide(quiet, {'status': 'completed', 'success': True}, {'reply': 'done'}) + assert decision['should_alert'] is True + assert decision['severity'] == 'info' + assert decision['delivery'] == 'notify_only' + + loud = build_workflow([ + {'id': 'r1', 'name': 'Loud', 'severity': 'high', + 'condition': {'type': 'run_status', 'statuses': ['completed']}}, + ]) + decision = decide(loud, {'status': 'completed', 'success': True}, {'reply': 'done'}) + assert decision['delivery'] == 'popup' + + override = build_workflow([ + {'id': 'r1', 'name': 'Quiet but loud', 'severity': 'info', 'delivery': 'popup', + 'condition': {'type': 'run_status', 'statuses': ['completed']}}, + ]) + decision = decide(override, {'status': 'completed', 'success': True}, {'reply': 'done'}) + assert decision['delivery'] == 'popup' + print("Severity ladder and delivery defaults passed.") + + +def test_no_matching_rule_produces_no_alert(): + """A run that matches nothing must stay completely silent.""" + print("Testing silent runs...") + workflow = build_workflow([ + {'id': 'r1', 'name': 'Only on failure', 'severity': 'high', + 'condition': {'type': 'run_status', 'statuses': ['failed']}}, + ]) + decision = decide(workflow, {'status': 'completed', 'success': True}, {'reply': 'nothing notable'}) + assert decision['should_alert'] is False + assert not decision['matched_rules'] + assert summarize_alert_decision(decision) == 'No alert rule matched this run.' + + off_workflow = build_workflow([ + {'id': 'r1', 'name': 'Anything', 'severity': 'high', + 'condition': {'type': 'run_status', 'statuses': ['completed']}}, + ], mode='off') + decision = decide(off_workflow, {'status': 'completed', 'success': True}, {'reply': 'x'}) + assert decision['should_alert'] is False + print("Silent runs passed.") + + +def test_highest_severity_wins_and_lists_every_match(): + """Every matched rule is reported, and the loudest one sets the severity.""" + print("Testing highest severity resolution...") + workflow = build_workflow([ + {'id': 'quiet', 'name': 'Ran', 'severity': 'info', + 'condition': {'type': 'run_status', 'statuses': ['completed']}}, + {'id': 'loud', 'name': 'Expiring found', 'severity': 'critical', + 'condition': {'type': 'text_match', 'mode': 'contains_any', 'values': ['EXPIRING']}}, + {'id': 'middle', 'name': 'Warning seen', 'severity': 'medium', + 'condition': {'type': 'text_match', 'mode': 'contains_any', 'values': ['warning']}}, + ]) + decision = decide( + workflow, + {'status': 'completed', 'success': True}, + {'reply': 'warning: 3 certificates EXPIRING soon'}, + ) + + assert decision['should_alert'] is True + assert decision['severity'] == 'critical' + assert decision['winning_rule_name'] == 'Expiring found' + matched_names = {match['rule_name'] for match in decision['matched_rules']} + assert matched_names == {'Ran', 'Expiring found', 'Warning seen'} + assert 'Expiring found' in summarize_alert_decision(decision) + print("Highest severity resolution passed.") + + +def test_failure_category_is_separate_from_severity(): + """Run and task errors are categorized as failures whatever severity the owner picked.""" + print("Testing failure category...") + workflow = build_workflow([ + {'id': 'r1', 'name': 'Run failed', 'severity': 'low', + 'condition': {'type': 'run_status', 'statuses': ['failed']}}, + ]) + decision = decide(workflow, {'status': 'failed', 'success': False, 'error': 'boom'}) + assert decision['should_alert'] is True + assert decision['severity'] == 'low' + assert decision['category'] == 'failure' + + task_workflow = build_workflow([ + {'id': 'r1', 'name': 'Task failed', 'severity': 'medium', + 'scope': {'type': 'any_task'}, + 'condition': {'type': 'task_status', 'statuses': ['failed']}}, + ]) + decision = decide( + task_workflow, + {'status': 'completed', 'success': True}, + { + 'reply': 'partial', + 'task_results': [ + {'task': {'id': 't1', 'name': 'Collect'}, 'status': 'failed', 'error': 'timeout', 'result': {}}, + {'task': {'id': 't2', 'name': 'Report'}, 'status': 'succeeded', 'result': {'reply': 'ok'}}, + ], + }, + ) + assert decision['should_alert'] is True + assert decision['category'] == 'failure' + assert 'Collect' in decision['matched_rules'][0]['reason'] + + success_workflow = build_workflow([ + {'id': 'r1', 'name': 'Found something', 'severity': 'critical', + 'condition': {'type': 'text_match', 'mode': 'contains_any', 'values': ['found']}}, + ]) + decision = decide(success_workflow, {'status': 'completed', 'success': True}, {'reply': 'found it'}) + assert decision['category'] == 'alert' + print("Failure category passed.") + + +def test_completed_with_task_errors_is_its_own_status(): + """A run that completes while a task errored can be alerted on separately.""" + print("Testing completed_with_task_errors...") + workflow = build_workflow([ + {'id': 'r1', 'name': 'Partial completion', 'severity': 'high', + 'condition': {'type': 'run_status', 'statuses': ['completed_with_task_errors']}}, + ]) + decision = decide( + workflow, + {'status': 'completed', 'success': True}, + { + 'reply': 'done', + 'task_error_count': 1, + 'task_results': [ + {'task': {'id': 't1', 'name': 'Collect'}, 'status': 'failed', 'error': 'nope', 'result': {}}, + ], + }, + ) + assert decision['should_alert'] is True + assert decision['category'] == 'failure' + + clean_decision = decide(workflow, {'status': 'completed', 'success': True}, {'reply': 'done'}) + assert clean_decision['should_alert'] is False + print("completed_with_task_errors passed.") + + +def test_text_match_modes_and_scoping(): + """Text conditions honor each match mode and the configured scope.""" + print("Testing text match modes and scoping...") + execution_result = { + 'reply': 'Summary: all systems nominal', + 'task_results': [ + {'task': {'id': 't1', 'name': 'Scan'}, 'status': 'succeeded', + 'result': {'reply': 'Detected 4 CRITICAL findings and 2 warnings'}}, + {'task': {'id': 't2', 'name': 'Summarize'}, 'status': 'succeeded', + 'result': {'reply': 'Summary: all systems nominal'}}, + ], + } + run_record = {'status': 'completed', 'success': True} + + # Final output does not contain CRITICAL, but a task output does. + final_scope = build_workflow([ + {'id': 'r1', 'name': 'Final', 'severity': 'high', 'scope': {'type': 'final'}, + 'condition': {'type': 'text_match', 'mode': 'contains_any', 'values': ['CRITICAL']}}, + ]) + assert decide(final_scope, run_record, execution_result)['should_alert'] is False + + any_task_scope = build_workflow([ + {'id': 'r1', 'name': 'Any task', 'severity': 'high', 'scope': {'type': 'any_task'}, + 'condition': {'type': 'text_match', 'mode': 'contains_any', 'values': ['CRITICAL']}}, + ]) + decision = decide(any_task_scope, run_record, execution_result) + assert decision['should_alert'] is True + assert 'Scan' in decision['matched_rules'][0]['reason'] + + specific_task_scope = build_workflow([ + {'id': 'r1', 'name': 'Only summarize', 'severity': 'high', + 'scope': {'type': 'task', 'task_id': 't2'}, + 'condition': {'type': 'text_match', 'mode': 'contains_any', 'values': ['CRITICAL']}}, + ]) + assert decide(specific_task_scope, run_record, execution_result)['should_alert'] is False + + contains_all = build_workflow([ + {'id': 'r1', 'name': 'Both terms', 'severity': 'high', 'scope': {'type': 'any_task'}, + 'condition': {'type': 'text_match', 'mode': 'contains_all', 'values': ['CRITICAL', 'warnings']}}, + ]) + assert decide(contains_all, run_record, execution_result)['should_alert'] is True + + contains_all_miss = build_workflow([ + {'id': 'r1', 'name': 'Both terms', 'severity': 'high', 'scope': {'type': 'any_task'}, + 'condition': {'type': 'text_match', 'mode': 'contains_all', 'values': ['CRITICAL', 'unrelated']}}, + ]) + assert decide(contains_all_miss, run_record, execution_result)['should_alert'] is False + + not_contains = build_workflow([ + {'id': 'r1', 'name': 'Missing signoff', 'severity': 'medium', 'scope': {'type': 'final'}, + 'condition': {'type': 'text_match', 'mode': 'not_contains', 'values': ['signed off']}}, + ]) + assert decide(not_contains, run_record, execution_result)['should_alert'] is True + + regex_rule = build_workflow([ + {'id': 'r1', 'name': 'Counted findings', 'severity': 'high', 'scope': {'type': 'any_task'}, + 'condition': {'type': 'text_match', 'mode': 'regex', 'pattern': r'\d+ CRITICAL'}}, + ]) + assert decide(regex_rule, run_record, execution_result)['should_alert'] is True + + case_sensitive = build_workflow([ + {'id': 'r1', 'name': 'Exact case', 'severity': 'high', 'scope': {'type': 'any_task'}, + 'condition': {'type': 'text_match', 'mode': 'contains_any', 'values': ['critical'], + 'case_sensitive': True}}, + ]) + assert decide(case_sensitive, run_record, execution_result)['should_alert'] is False + print("Text match modes and scoping passed.") + + +def test_file_sync_no_output_and_agent_signal_conditions(): + """File Sync outcomes, empty output and agent raised signals all drive alerts.""" + print("Testing file sync, no output and agent signal conditions...") + changes_rule = build_workflow([ + {'id': 'r1', 'name': 'New documents', 'severity': 'medium', + 'condition': {'type': 'file_sync', 'outcome': 'changes_found'}}, + ]) + decision = decide( + changes_rule, + {'status': 'completed', 'success': True, 'file_sync': {'changed_documents': [{'id': 'doc-1'}]}}, + {'reply': 'ok'}, + ) + assert decision['should_alert'] is True + assert decision['category'] == 'alert' + + decision = decide( + changes_rule, + {'status': 'completed', 'success': True, 'file_sync': {'changed_documents': []}}, + {'reply': 'ok'}, + ) + assert decision['should_alert'] is False + + sync_failed_rule = build_workflow([ + {'id': 'r1', 'name': 'Sync broke', 'severity': 'high', + 'condition': {'type': 'file_sync', 'outcome': 'sync_failed'}}, + ]) + decision = decide( + sync_failed_rule, + {'status': 'completed', 'success': True, 'file_sync': {'error': 'blob unreachable'}}, + {'reply': 'ok'}, + ) + assert decision['should_alert'] is True + assert decision['category'] == 'failure' + + no_output_rule = build_workflow([ + {'id': 'r1', 'name': 'Nothing came back', 'severity': 'medium', + 'condition': {'type': 'no_output'}}, + ]) + assert decide(no_output_rule, {'status': 'completed', 'success': True}, {'reply': ''})['should_alert'] is True + assert decide(no_output_rule, {'status': 'completed', 'success': True}, {'reply': 'x'})['should_alert'] is False + + signal_rule = build_workflow([ + {'id': 'r1', 'name': 'Agent flagged it', 'severity': 'low', + 'condition': {'type': 'agent_signal', 'min_severity': 'medium'}}, + ]) + decision = decide( + signal_rule, + {'status': 'completed', 'success': True}, + { + 'reply': 'ok', + 'agent_alert_signals': [ + {'severity': 'critical', 'title': 'Breach detected', 'reason': 'Unrecognized sign-in from a new region.'}, + ], + }, + ) + assert decision['should_alert'] is True + # The agent can escalate above the rule floor but never below it. + assert decision['severity'] == 'critical' + assert 'Unrecognized sign-in' in decision['matched_rules'][0]['reason'] + + decision = decide( + signal_rule, + {'status': 'completed', 'success': True}, + {'reply': 'ok', 'agent_alert_signals': [{'severity': 'info', 'title': 'FYI'}]}, + ) + assert decision['should_alert'] is False + + named_signal_rule = build_workflow([ + {'id': 'r1', 'name': 'Only cert signals', 'severity': 'high', + 'condition': {'type': 'agent_signal', 'signal_name': 'expiring-certificates', 'min_severity': 'info'}}, + ]) + decision = decide( + named_signal_rule, + {'status': 'completed', 'success': True}, + {'reply': 'ok', 'agent_alert_signals': [{'severity': 'high', 'signal_name': 'other-signal'}]}, + ) + assert decision['should_alert'] is False + print("File sync, no output and agent signal conditions passed.") + + +def test_legacy_alert_priority_migration_preserves_behavior(): + """Workflows with only alert_priority keep alerting through editable rules.""" + print("Testing legacy migration...") + legacy_workflow = {'id': 'w', 'name': 'Legacy', 'alert_priority': 'low'} + config = resolve_workflow_alert_config(legacy_workflow) + assert config['alert_mode'] == 'rules' + assert [rule['name'] for rule in config['alert_rules']] == ['Run failed', 'Run completed'] + assert config['alert_rules'][0]['severity'] == 'high' + assert config['alert_rules'][1]['severity'] == 'low' + # Legacy alerts always opened the modal, so migration pins delivery to popup. + assert all(rule['delivery'] == 'popup' for rule in config['alert_rules']) + + decision = decide(legacy_workflow, {'status': 'completed', 'success': True}, {'reply': 'x'}) + assert decision['should_alert'] is True + assert decision['severity'] == 'low' + assert decision['delivery'] == 'popup' + + decision = decide(legacy_workflow, {'status': 'failed', 'success': False, 'error': 'boom'}) + assert decision['severity'] == 'high' + assert decision['category'] == 'failure' + + # Cancelled runs stayed silent before and must stay silent now. + decision = decide(legacy_workflow, {'status': 'cancelled', 'success': False}) + assert decision['should_alert'] is False + + silent_legacy = resolve_workflow_alert_config({'id': 'w', 'alert_priority': 'none'}) + assert silent_legacy['alert_mode'] == 'off' + assert build_legacy_alert_rules('none') == [] + print("Legacy migration passed.") + + +def test_every_run_mode_matches_previous_behavior(): + """The legacy every-run mode remains selectable and behaves as before.""" + print("Testing every_run mode...") + workflow = {'id': 'w', 'name': 'Noisy', 'alert_mode': 'every_run', 'alert_priority': 'medium'} + decision = decide(workflow, {'status': 'completed', 'success': True}, {'reply': 'x'}) + assert decision['should_alert'] is True + assert decision['severity'] == 'medium' + assert decision['delivery'] == 'popup' + assert decision['category'] == 'alert' + + decision = decide(workflow, {'status': 'failed', 'success': False, 'error': 'boom'}) + assert decision['category'] == 'failure' + + decision = decide(workflow, {'status': 'cancelled', 'success': False}) + assert decision['should_alert'] is False + print("every_run mode passed.") + + +def test_rule_validation_rejects_bad_configuration(): + """Invalid rules fail at save time rather than silently misbehaving at run time.""" + print("Testing rule validation...") + for bad_pattern in ['(a+)+', '([a-z]*)*', 'x' * 500, '(unclosed']: + try: + validate_alert_regex(bad_pattern) + raise AssertionError(f'Expected regex rejection for {bad_pattern!r}') + except ValueError: + pass + assert validate_alert_regex(r'\d+ CRITICAL') is not None + + def expect_value_error(callback, label): + try: + callback() + except ValueError: + return + raise AssertionError(f'Expected ValueError for {label}') + + expect_value_error( + lambda: normalize_alert_rules([{'condition': {'type': 'unknown_condition'}}]), + 'unknown condition type', + ) + expect_value_error( + lambda: normalize_alert_rules([{'severity': 'urgent', 'condition': {'type': 'no_output'}}]), + 'unsupported severity', + ) + expect_value_error( + lambda: normalize_alert_rules([{'condition': {'type': 'run_status', 'statuses': ['exploded']}}]), + 'unsupported run status', + ) + expect_value_error( + lambda: normalize_alert_rules([{'condition': {'type': 'model_evaluation', 'prompt': ''}}]), + 'empty model evaluation prompt', + ) + expect_value_error( + lambda: normalize_alert_rules( + [{'scope': {'type': 'task', 'task_id': 'missing'}, 'condition': {'type': 'no_output'}}], + task_ids=['t1'], + ), + 'task scope referencing an unknown task', + ) + expect_value_error( + lambda: normalize_workflow_alert_settings({'alert_mode': 'rules', 'alert_rules': []}), + 'rules mode without any rules', + ) + expect_value_error( + lambda: normalize_workflow_alert_settings({'alert_mode': 'every_run', 'alert_priority': 'none'}), + 'every_run mode without a priority', + ) + expect_value_error( + lambda: normalize_workflow_alert_settings({'alert_mode': 'sometimes'}), + 'unknown alert mode', + ) + + # A rule without an explicit name is described from its condition. + normalized = normalize_alert_rules([{'condition': {'type': 'run_status', 'statuses': ['failed']}}]) + assert normalized[0]['name'] == 'Run status is failed' + assert normalized[0]['id'] + print("Rule validation passed.") + + +def test_save_settings_keeps_legacy_clients_alerting(): + """Clients that only send alert_priority still end up with working rules.""" + print("Testing save settings compatibility...") + settings = normalize_workflow_alert_settings({'alert_priority': 'high'}) + assert settings['alert_mode'] == 'rules' + assert settings['alert_priority'] == 'high' + assert len(settings['alert_rules']) == 2 + + settings = normalize_workflow_alert_settings({'alert_priority': 'none'}) + assert settings['alert_mode'] == 'off' + assert settings['alert_rules'] == [] + + # Clearing every rule without picking a mode turns alerts off instead of failing. + settings = normalize_workflow_alert_settings( + {'alert_rules': []}, + existing_workflow={'alert_mode': 'rules', 'alert_rules': [ + {'id': 'r1', 'severity': 'high', 'condition': {'type': 'no_output'}}, + ]}, + ) + assert settings['alert_mode'] == 'off' + + settings = normalize_workflow_alert_settings({'alert_evaluation': {'on_error': 'alert'}, + 'alert_mode': 'off'}) + assert settings['alert_evaluation']['on_error'] == 'alert' + print("Save settings compatibility passed.") + + +def test_alert_rule_contracts_are_wired_through_the_stack(): + """The rule engine is referenced by the save paths, runner, UI and notifications.""" + print("Testing stack wiring contracts...") + assert_app_version_at_least("0.250.213") + + personal_content = read_text("application/single_app/functions_personal_workflows.py") + group_content = read_text("application/single_app/functions_group_workflows.py") + runner_content = read_text("application/single_app/functions_workflow_runner.py") + notifications_content = read_text("application/single_app/functions_notifications.py") + activity_content = read_text("application/single_app/functions_workflow_activity.py") + workspace_template = read_text("application/single_app/templates/workspace.html") + group_template = read_text("application/single_app/templates/group_workspaces.html") + workflow_js = read_text("application/single_app/static/js/workspace/workspace_workflows.js") + notifications_js = read_text("application/single_app/static/js/notifications.js") + base_template = read_text("application/single_app/templates/base.html") + + assert 'from functions_workflow_alerts import normalize_workflow_alert_settings' in personal_content + assert "'alert_rules': alert_settings['alert_rules']," in personal_content + assert 'from functions_workflow_alerts import normalize_workflow_alert_settings' in group_content + assert "'alert_rules': alert_settings['alert_rules']," in group_content + + assert 'from functions_workflow_alerts import (' in runner_content + assert 'def _workflow_alert_rules_need_model_evaluation(alert_config, facts):' in runner_content + assert 'def _build_workflow_alert_model_evaluator(workflow, settings=None):' in runner_content + assert "if not decision.get('should_alert'):" in runner_content + assert "'category': decision.get('category') or 'alert'," in runner_content + assert "'delivery': decision.get('delivery') or 'popup'," in runner_content + assert 'def _record_workflow_alert_decision(workflow, run_record, decision):' in runner_content + + assert "'info': {" in notifications_content + assert "'critical': {" in notifications_content + assert 'def _get_workflow_alert_category(notification):' in notifications_content + assert 'def _get_workflow_alert_delivery(notification):' in notifications_content + assert 'WORKFLOW_ALERT_DELIVERY_NOTIFY_ONLY' in notifications_content + + assert 'resolve_workflow_alert_config' in activity_content + assert "'alert_rules': [" in activity_content + + for template in (workspace_template, group_template): + assert 'id="workflow-alert-mode"' in template + assert 'id="workflow-alert-rules-list"' in template + assert 'id="workflow-alert-rule-add-btn"' in template + assert 'id="workflow-alert-evaluation-on-error"' in template + + assert 'function renderWorkflowAlertRules()' in workflow_js + assert 'function collectWorkflowAlertRulesPayload()' in workflow_js + assert 'function validateWorkflowAlertRules()' in workflow_js + assert 'alert_mode: getWorkflowAlertMode(),' in workflow_js + assert 'alert_rules: collectWorkflowAlertRulesPayload(),' in workflow_js + assert 'function buildLegacyWorkflowAlertRules(priority)' in workflow_js + + assert "normalizedPriority === 'critical'" in notifications_js + assert "normalizedPriority === 'info'" in notifications_js + assert 'function getWorkflowAlertDelivery(notification)' in notifications_js + assert "getWorkflowAlertDelivery(notification) === 'notify_only'" in notifications_js + assert '[data-priority="critical"]' in base_template + assert '[data-category="failure"]' in base_template + print("Stack wiring contracts passed.") + + +if __name__ == '__main__': + tests = [ + test_severity_ladder_and_delivery_defaults, + test_no_matching_rule_produces_no_alert, + test_highest_severity_wins_and_lists_every_match, + test_failure_category_is_separate_from_severity, + test_completed_with_task_errors_is_its_own_status, + test_text_match_modes_and_scoping, + test_file_sync_no_output_and_agent_signal_conditions, + test_legacy_alert_priority_migration_preserves_behavior, + test_every_run_mode_matches_previous_behavior, + test_rule_validation_rejects_bad_configuration, + test_save_settings_keeps_legacy_clients_alerting, + test_alert_rule_contracts_are_wired_through_the_stack, + ] + + results = [] + for test in tests: + try: + test() + results.append(True) + except Exception as exc: + print(f"Test failed: {test.__name__}: {exc}") + import traceback + traceback.print_exc() + results.append(False) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_workflow_priority_alerts.py b/functional_tests/test_workflow_priority_alerts.py index 21beddf70..a2a8ddde2 100644 --- a/functional_tests/test_workflow_priority_alerts.py +++ b/functional_tests/test_workflow_priority_alerts.py @@ -52,7 +52,7 @@ def test_workflow_priority_alert_contracts(): assert "default_label='Open workflow'" in workflow_runner_content assert 'def _summarize_workflow_alert_text(text, max_length=140):' in workflow_runner_content assert 'def _select_preferred_workflow_alert_targets(targets):' in workflow_runner_content - assert 'def _build_workflow_alert_content(workflow, run_record, execution_result, priority):' in workflow_runner_content + assert 'def _build_workflow_alert_content(workflow, run_record, execution_result, priority, decision=None):' in workflow_runner_content assert "'alert_summary': alert_content.get('alert_summary')," in workflow_runner_content assert "'alert_detail': alert_content.get('alert_detail')," in workflow_runner_content assert 'plugin_logger.clear_invocations_for_conversation(user_id, conversation_id)' in workflow_runner_content diff --git a/ui_tests/test_workflow_priority_alert_modal.py b/ui_tests/test_workflow_priority_alert_modal.py index 5afa95964..2c57f7571 100644 --- a/ui_tests/test_workflow_priority_alert_modal.py +++ b/ui_tests/test_workflow_priority_alert_modal.py @@ -1,13 +1,15 @@ # test_workflow_priority_alert_modal.py """ UI test for the workflow priority alert modal. -Version: 0.241.182 +Version: 0.250.213 Implemented in: 0.241.055 This test ensures unread workflow alerts open in the global modal, show the -configured priority, prefer alert-focused enrichment copy over legacy preview +configured severity, prefer alert-focused enrichment copy over legacy preview text, render linked conversations, open conversation targets in a new tab, and -can be marked as read from the browser workflow. +can be marked as read from the browser workflow. It also covers the expanded +severity ladder, the separate failure styling, and quiet alerts that must stay +in the notification bell instead of interrupting. """ import json @@ -208,4 +210,250 @@ def handle_mark_read(route): assert any(request_url.endswith('/api/notifications/workflow-alert-001/read') for request_url in read_requests) finally: context.close() - browser.close() \ No newline at end of file + browser.close() + +def _build_alert_notification(notification_id, priority, category, delivery, title, message): + """Build a minimal workflow alert payload for severity and delivery checks.""" + return { + "id": notification_id, + "title": title, + "message": message, + "created_at": "2025-01-01T10:00:00+00:00", + "priority": priority, + "category": category, + "delivery": delivery, + "link_url": "", + "link_context": {}, + "metadata": { + "workflow_name": "Certificate watch", + "priority": priority, + "category": category, + "delivery": delivery, + "trigger_source": "scheduled", + "alert_title": title, + "alert_summary": message, + "alert_detail": message, + "matched_rules": [ + { + "rule_id": "rule-1", + "rule_name": "Expiring certificates", + "severity": priority, + "condition_type": "model_evaluation", + "reason": "Two certificates expire in 3 days.", + } + ], + }, + "type_config": {"icon": "bi-exclamation-octagon", "color": "danger"}, + "is_read": False, + "is_dismissed": False, + } + + +@pytest.mark.ui +def test_workflow_alert_modal_renders_critical_severity(): + """Critical alerts use the loudest badge and accent.""" + _require_ui_env() + playwright_sync = _require_playwright() + + payload = { + "success": True, + "notifications": [ + _build_alert_notification( + "workflow-alert-critical", + "critical", + "alert", + "popup", + "Critical priority workflow alert: Expiring certificates", + "Two certificates expire in 3 days.", + ) + ], + } + + with playwright_sync.sync_playwright() as playwright: + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + + page.route("**/api/notifications/count", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True, "count": 1}), + )) + page.route("**/api/notifications/workflow-alerts**", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps(payload), + )) + page.route("**/api/notifications?*", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({ + "success": True, + "notifications": [], + "total": 0, + "page": 1, + "per_page": 20, + "has_more": False, + }), + )) + + try: + response = page.goto(f"{BASE_URL}/notifications", wait_until="networkidle") + assert response is not None, "Expected a navigation response when loading /notifications." + if response.status in SKIP_RESPONSE_CODES: + pytest.skip(f"Notifications page unavailable in this environment (HTTP {response.status}).") + assert response.ok, f"Expected /notifications to load successfully, got HTTP {response.status}." + + page.evaluate("window.dispatchEvent(new CustomEvent('workflow-alert-refresh-requested'))") + + modal = page.locator("#workflowAlertModal") + expect(modal).to_be_visible() + expect(page.locator("#workflow-alert-priority-badge")).to_have_text("CRITICAL PRIORITY") + expect(page.locator("#workflow-alert-priority-badge")).to_have_class(re.compile(r"text-bg-danger")) + expect(page.locator("#workflowAlertModal .workflow-alert-modal-content")).to_have_attribute( + "data-priority", "critical" + ) + finally: + context.close() + browser.close() + + +@pytest.mark.ui +def test_workflow_alert_modal_marks_run_failures_distinctly(): + """A failed run is labelled as a failure regardless of its severity.""" + _require_ui_env() + playwright_sync = _require_playwright() + + payload = { + "success": True, + "notifications": [ + _build_alert_notification( + "workflow-alert-failure", + "low", + "failure", + "popup", + "Low priority workflow alert: Certificate watch failed", + "The run could not reach the certificate service.", + ) + ], + } + + with playwright_sync.sync_playwright() as playwright: + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + + page.route("**/api/notifications/count", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True, "count": 1}), + )) + page.route("**/api/notifications/workflow-alerts**", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps(payload), + )) + page.route("**/api/notifications?*", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({ + "success": True, + "notifications": [], + "total": 0, + "page": 1, + "per_page": 20, + "has_more": False, + }), + )) + + try: + response = page.goto(f"{BASE_URL}/notifications", wait_until="networkidle") + assert response is not None, "Expected a navigation response when loading /notifications." + if response.status in SKIP_RESPONSE_CODES: + pytest.skip(f"Notifications page unavailable in this environment (HTTP {response.status}).") + assert response.ok, f"Expected /notifications to load successfully, got HTTP {response.status}." + + page.evaluate("window.dispatchEvent(new CustomEvent('workflow-alert-refresh-requested'))") + + modal = page.locator("#workflowAlertModal") + expect(modal).to_be_visible() + expect(page.locator("#workflow-alert-priority-badge")).to_have_text("LOW FAILURE") + expect(page.locator("#workflowAlertModal .workflow-alert-modal-content")).to_have_attribute( + "data-category", "failure" + ) + finally: + context.close() + browser.close() + + +@pytest.mark.ui +def test_quiet_workflow_alerts_do_not_open_the_modal(): + """Notification-only alerts stay in the bell instead of interrupting.""" + _require_ui_env() + playwright_sync = _require_playwright() + + payload = { + "success": True, + "notifications": [ + _build_alert_notification( + "workflow-alert-quiet", + "info", + "alert", + "notify_only", + "Info priority workflow alert: Nightly summary", + "The nightly summary completed with nothing notable.", + ) + ], + } + + with playwright_sync.sync_playwright() as playwright: + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + + page.route("**/api/notifications/count", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True, "count": 1}), + )) + page.route("**/api/notifications/workflow-alerts**", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps(payload), + )) + page.route("**/api/notifications?*", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({ + "success": True, + "notifications": [], + "total": 0, + "page": 1, + "per_page": 20, + "has_more": False, + }), + )) + + try: + response = page.goto(f"{BASE_URL}/notifications", wait_until="networkidle") + assert response is not None, "Expected a navigation response when loading /notifications." + if response.status in SKIP_RESPONSE_CODES: + pytest.skip(f"Notifications page unavailable in this environment (HTTP {response.status}).") + assert response.ok, f"Expected /notifications to load successfully, got HTTP {response.status}." + + page.evaluate("window.dispatchEvent(new CustomEvent('workflow-alert-refresh-requested'))") + page.wait_for_timeout(1000) + + expect(page.locator("#workflowAlertModal")).not_to_be_visible() + finally: + context.close() + browser.close() diff --git a/ui_tests/test_workspace_workflow_alert_rules.py b/ui_tests/test_workspace_workflow_alert_rules.py new file mode 100644 index 000000000..baa839b77 --- /dev/null +++ b/ui_tests/test_workspace_workflow_alert_rules.py @@ -0,0 +1,246 @@ +# test_workspace_workflow_alert_rules.py +""" +UI test for the workflow alert rules editor. +Version: 0.250.213 +Implemented in: 0.250.213 + +This test ensures the workflow modal exposes the alert mode selector, shows the +rules editor only in rules mode, lets an owner build a condition-based rule, and +sends alert_mode, alert_rules and alert_evaluation in the save payload. It also +verifies a legacy workflow carrying only alert_priority loads as editable +migrated rules. +""" + +import json +import os +import re +from pathlib import Path + +import pytest + + +BASE_URL = os.getenv("SIMPLECHAT_UI_BASE_URL", "").rstrip("/") +STORAGE_STATE = os.getenv("SIMPLECHAT_UI_STORAGE_STATE", "") + + +def _get_playwright_sync(): + return pytest.importorskip("playwright.sync_api", reason="Install Playwright to run this UI test.") + + +def _require_ui_env(): + if not BASE_URL: + pytest.skip("Set SIMPLECHAT_UI_BASE_URL to run this UI test.") + if not STORAGE_STATE or not Path(STORAGE_STATE).exists(): + pytest.skip("Set SIMPLECHAT_UI_STORAGE_STATE to a valid authenticated Playwright storage state file.") + + +def _build_workflow_state(): + return { + "items": [ + { + "id": "workflow-legacy-1", + "name": "Legacy Noisy Workflow", + "description": "Created before alert rules existed.", + "task_prompt": "Summarize the latest documents.", + "runner_type": "model", + "trigger_type": "manual", + "is_enabled": True, + "model_binding_summary": {"label": "Default app model"}, + "alert_priority": "medium", + "status": "idle", + "tasks": [ + { + "id": "task-1", + "type": "instructions", + "name": "Summarize", + "instructions": "Summarize the latest documents.", + "order": 1, + "runner": {"type": "inherit"}, + } + ], + }, + ], + "saved_payloads": [], + } + + +def _route_workflow_api(page, workflow_state): + def handler(route): + request = route.request + method = request.method + + if method == "GET": + route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"workflows": workflow_state["items"]}), + ) + return + + if method in {"POST", "PUT", "PATCH"}: + payload = json.loads(request.post_data or "{}") + workflow_state["saved_payloads"].append(payload) + saved_workflow = { + "id": payload.get("id") or "workflow-new", + "name": payload.get("name"), + "description": payload.get("description"), + "task_prompt": payload.get("task_prompt"), + "runner_type": payload.get("runner_type"), + "trigger_type": payload.get("trigger_type"), + "alert_mode": payload.get("alert_mode"), + "alert_rules": payload.get("alert_rules", []), + "alert_priority": payload.get("alert_priority"), + "is_enabled": payload.get("is_enabled", True), + "status": "idle", + } + workflow_state["items"] = [saved_workflow, *workflow_state["items"]] + route.fulfill( + status=201, + content_type="application/json", + body=json.dumps({"success": True, "workflow": saved_workflow}), + ) + return + + route.fulfill(status=405, content_type="application/json", body=json.dumps({"error": "Unsupported"})) + + page.route("**/api/user/workflows**", handler) + + +def _route_agent_api(page): + page.route("**/api/user/agents", lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps([]), + )) + + +def _open_workflows_tab(page, expect): + if page.locator("#workflows-tab-btn").count() == 0: + pytest.skip("Personal workflows are disabled or unavailable for this authenticated user.") + page.locator("#workflows-tab-btn").evaluate("button => button.click()") + expect(page.locator("#workflows-tab")).to_be_visible() + + +@pytest.mark.ui +def test_workflow_alert_rules_editor_builds_a_conditional_alert(): + """An owner can choose rules mode and save a condition-based alert rule.""" + _require_ui_env() + playwright_sync = _get_playwright_sync() + expect = playwright_sync.expect + playwright_manager = playwright_sync.sync_playwright() + playwright = playwright_manager.start() + + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + workflow_state = _build_workflow_state() + + _route_workflow_api(page, workflow_state) + _route_agent_api(page) + + try: + response = page.goto(f"{BASE_URL}/workspace", wait_until="networkidle") + assert response is not None, "Expected a navigation response when loading /workspace." + assert response.ok, f"Expected /workspace to load successfully, got HTTP {response.status}." + + _open_workflows_tab(page, expect) + + page.get_by_role("button", name="New Workflow").click() + expect(page.locator("#workflowModal")).to_be_visible() + page.fill("#workflow-name", "Certificate Watch") + page.fill("#workflow-description", "Alert only when certificates are expiring.") + page.fill("#workflow-task-prompt", "List certificates expiring in the next 30 days.") + + # Alerts default to off, so neither the priority nor the rules editor shows. + expect(page.locator("#workflow-alert-mode")).to_have_value("off") + expect(page.locator("#workflow-alert-rules-group")).to_have_class(re.compile(r"\bd-none\b")) + + page.select_option("#workflow-alert-mode", "rules") + expect(page.locator("#workflow-alert-rules-group")).not_to_have_class(re.compile(r"\bd-none\b")) + expect(page.locator("#workflow-alert-priority-group")).to_have_class(re.compile(r"\bd-none\b")) + + # Switching to rules seeds a starter rule so the editor is never empty. + rule_rows = page.locator("#workflow-alert-rules-list .workflow-alert-rule") + expect(rule_rows).to_have_count(1) + + page.locator("#workflow-alert-rule-add-btn").click() + expect(rule_rows).to_have_count(2) + + second_rule = rule_rows.nth(1) + second_rule.locator("input[type='text']").first.fill("Expiring certificates") + second_rule.locator("select").nth(0).select_option("text_match") + second_rule.locator("select").nth(1).select_option("critical") + second_rule.locator("input[type='text']").last.fill("EXPIRING") + + page.click("#workflow-save-btn") + + assert workflow_state["saved_payloads"], "Expected the save handler to capture the workflow payload." + saved_payload = workflow_state["saved_payloads"][-1] + assert saved_payload["alert_mode"] == "rules" + assert isinstance(saved_payload["alert_rules"], list) + assert len(saved_payload["alert_rules"]) == 2 + assert saved_payload["alert_evaluation"]["on_error"] == "skip" + + rule_names = [rule["name"] for rule in saved_payload["alert_rules"]] + assert "Expiring certificates" in rule_names + + text_rule = next( + rule for rule in saved_payload["alert_rules"] + if rule["condition"]["type"] == "text_match" + ) + assert text_rule["severity"] == "critical" + assert text_rule["condition"]["values"] == ["EXPIRING"] + assert text_rule["scope"]["type"] == "final" + finally: + context.close() + browser.close() + playwright_manager.stop() + + +@pytest.mark.ui +def test_legacy_workflow_loads_as_editable_migrated_rules(): + """A workflow carrying only alert_priority opens with its migrated rules.""" + _require_ui_env() + playwright_sync = _get_playwright_sync() + expect = playwright_sync.expect + playwright_manager = playwright_sync.sync_playwright() + playwright = playwright_manager.start() + + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + workflow_state = _build_workflow_state() + + _route_workflow_api(page, workflow_state) + _route_agent_api(page) + + try: + response = page.goto(f"{BASE_URL}/workspace", wait_until="networkidle") + assert response is not None, "Expected a navigation response when loading /workspace." + assert response.ok, f"Expected /workspace to load successfully, got HTTP {response.status}." + + _open_workflows_tab(page, expect) + + legacy_row = page.locator("#workflows-table-body tr").filter(has_text="Legacy Noisy Workflow") + expect(legacy_row).to_be_visible() + expect(legacy_row).to_contain_text("Alert: Every run (medium)") + + legacy_row.get_by_role("button", name="Edit").click() + expect(page.locator("#workflowModal")).to_be_visible() + + # The legacy priority is materialized as two editable rules. + expect(page.locator("#workflow-alert-mode")).to_have_value("rules") + rule_rows = page.locator("#workflow-alert-rules-list .workflow-alert-rule") + expect(rule_rows).to_have_count(2) + expect(page.locator("#workflow-alert-rules-list")).to_contain_text("Run failed") + expect(page.locator("#workflow-alert-rules-list")).to_contain_text("Run completed") + finally: + context.close() + browser.close() + playwright_manager.stop() diff --git a/ui_tests/test_workspace_workflows_tab.py b/ui_tests/test_workspace_workflows_tab.py index a1e9f4e4e..2d3dbbfda 100644 --- a/ui_tests/test_workspace_workflows_tab.py +++ b/ui_tests/test_workspace_workflows_tab.py @@ -1,14 +1,14 @@ # test_workspace_workflows_tab.py """ UI test for personal workflows workspace tab. -Version: 0.241.189 +Version: 0.250.213 Implemented in: 0.241.029 This test ensures the personal workspace workflows tab renders the left-hand menu entry, shows workflow rows, opens run history with direct workflow -conversation links, and submits new agent-based interval workflows with alert -priorities from the modal on desktop and mobile viewports. It also verifies the -Activity action opens only in a new tab. +conversation links, and submits new agent-based interval workflows with the +selected alert mode from the modal on desktop and mobile viewports. It also +verifies the Activity action opens only in a new tab. """ import json @@ -204,7 +204,7 @@ def test_workspace_workflows_tab_desktop(): expect(summary_row).to_be_visible() expect(summary_row).to_contain_text("Default app model") expect(summary_row).to_contain_text("Every 10 seconds") - expect(summary_row).to_contain_text("Alert: Low priority") + expect(summary_row).to_contain_text("Alert: Every run (low)") expect(summary_row).to_contain_text("Digest completed.") current_url = page.url @@ -235,6 +235,7 @@ def test_workspace_workflows_tab_desktop(): page.select_option("#workflow-trigger-type", "interval") page.fill("#workflow-schedule-value", "10") page.select_option("#workflow-schedule-unit", "seconds") + page.select_option("#workflow-alert-mode", "every_run") page.select_option("#workflow-alert-priority", "high") page.click("#workflow-save-btn") @@ -243,6 +244,7 @@ def test_workspace_workflows_tab_desktop(): saved_payload = workflow_state["saved_payloads"][0] assert saved_payload["runner_type"] == "agent" assert saved_payload["trigger_type"] == "interval" + assert saved_payload["alert_mode"] == "every_run" assert saved_payload["alert_priority"] == "high" assert saved_payload["schedule"] == {"value": 10, "unit": "seconds"} assert saved_payload["selected_agent"]["name"] == "researcher_agent"