diff --git a/application/single_app/background_tasks.py b/application/single_app/background_tasks.py index b4fdf5228..bc4c64c57 100644 --- a/application/single_app/background_tasks.py +++ b/application/single_app/background_tasks.py @@ -921,6 +921,45 @@ def run_orchestration_scheduler_loop(app=None): time.sleep(30) +def run_workflow_chat_delivery_loop(app=None): + """Post chat-started workflow outcomes back into the chats that asked for them. + + A hint from the runtime projection wakes the loop at once. The cross-user sweep runs at most + once per sweep interval, and the worker runs it only while it holds the distributed lock. + """ + # Deferred like the orchestration loop: the worker binds application storage on first use. + import functions_workflow_chat_delivery as delivery_contract + import functions_workflow_chat_delivery_worker as delivery_worker + + interval = delivery_contract.SWEEP_INTERVAL_SECONDS + services = None + next_sweep_at = 0.0 + while True: + started = time.monotonic() + sweep_due = started >= next_sweep_at + if sweep_due: + next_sweep_at = started + interval + try: + if services is None: + services = delivery_worker.default_workflow_chat_delivery_services( + acquire_lock=acquire_distributed_task_lock, + release_lock=release_distributed_task_lock, + ) + with app.app_context() if app is not None else nullcontext(): + delivery_worker.process_workflow_chat_delivery_hints(services) + if sweep_due: + delivery_worker.run_workflow_chat_delivery_sweep(services) + except Exception as exc: + log_event( + '[WORKFLOW_CHAT_DELIVERY] Delivery slice could not complete.', + level=logging.ERROR, extra={'error_type': type(exc).__name__}, + ) + # A failure can leave hints queued, so wait without them to avoid a busy loop. + time.sleep(interval) + continue + delivery_contract.wait_for_workflow_chat_delivery_hint(max(1.0, next_sweep_at - time.monotonic())) + + def run_data_management_scheduler_loop(app=None): """Queue due backup and recoverable migration jobs across scaled-out workers.""" while True: @@ -1045,6 +1084,7 @@ def start_background_task_threads(app=None): ('File Sync scheduler background task started.', run_file_sync_scheduler_loop), ('Tabular generated-output scheduler background task started.', run_tabular_generated_output_scheduler_loop), ('Orchestration scheduler background task started.', lambda: run_orchestration_scheduler_loop(app=app)), + ('Workflow chat delivery background task started.', lambda: run_workflow_chat_delivery_loop(app=app)), ('Data Management scheduler background task started.', lambda: run_data_management_scheduler_loop(app=app)), ('Content screening scheduler background task started.', lambda: run_content_screening_scheduler_loop(app=app)), ('App maintenance background task started.', run_app_maintenance_loop), diff --git a/application/single_app/config.py b/application/single_app/config.py index bf5a6c6e2..1fe91e5f0 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.226" +VERSION = "0.261.227" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_conversation_unread.py b/application/single_app/functions_conversation_unread.py index 583c4080a..31d8d33e8 100644 --- a/application/single_app/functions_conversation_unread.py +++ b/application/single_app/functions_conversation_unread.py @@ -2,7 +2,19 @@ """Helpers for conversation unread assistant-response state.""" -from datetime import datetime +from datetime import datetime, timezone + +from azure.core import MatchConditions +from azure.cosmos import exceptions + + +# Closed outcomes of mark_conversation_unread_guarded. +UNREAD_MARKED = 'marked' +UNREAD_ALREADY_MARKED = 'already_marked' +UNREAD_READ_SINCE = 'read_since' +UNREAD_NOT_FOUND = 'not_found' +UNREAD_UNAVAILABLE = 'unavailable' +UNREAD_CONFLICT = 'conflict' def normalize_conversation_unread_state(conversation_item): @@ -42,3 +54,101 @@ def clear_conversation_unread(conversation_item): normalized_item['last_unread_assistant_message_id'] = None normalized_item['last_unread_assistant_at'] = None return normalized_item + + +def _parse_utc_timestamp(value): + if not isinstance(value, str) or not value.strip(): + return None + try: + parsed = datetime.fromisoformat(value.strip().replace('Z', '+00:00')) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _reached(value, threshold): + """Whether timestamp ``value`` is at or after ``threshold``; False when either is unreadable.""" + parsed_value = _parse_utc_timestamp(value) + parsed_threshold = _parse_utc_timestamp(threshold) + return parsed_value is not None and parsed_threshold is not None and parsed_value >= parsed_threshold + + +def _conversation_unavailable(conversation_item, user_id, require_private): + if not isinstance(conversation_item, dict) or conversation_item.get('user_id') != user_id: + return True + if conversation_item.get('orchestration_deleted') or conversation_item.get('deleted'): + return True + if not require_private: + return False + # Imported here so the plain unread helpers above keep their light import graph. + from functions_orchestration_memory import conversation_is_private + + return not conversation_is_private(conversation_item, user_id) + + +def mark_conversation_unread_guarded( + container, + conversation_id, + user_id, + message_id, + planned_at, + *, + require_private=True, + skip_if_read_since=None, + max_retries=5, +): + """Mark one assistant message unread on a conversation, under an ETag guard. + + Every attempt rereads the conversation and rechecks that it exists, belongs to ``user_id``, + isn't being deleted and, by default, is still private, so a concurrent delete or share wins + over the mark. ``last_updated`` is raised to at least ``planned_at`` so the conversation sorts + with its newest message. + + ``skip_if_read_since`` is the planned time of an earlier attempt. Marking a chat read keeps its + ``last_updated``, so a conversation that already reached that time was either marked and then + read, or moved on without the mark; either way it is left alone. + + Returns ``(outcome, conversation)``. The outcome is one of the ``UNREAD_*`` constants and the + conversation is the stored item when one was read. The caller invalidates any conversation + cache after ``UNREAD_MARKED``. Storage errors other than a missing item or an ETag conflict + propagate. + """ + # An empty user id must never match a conversation that has no owner, so every id is required. + required = (conversation_id, user_id, message_id, planned_at) + if not all(isinstance(value, str) and value for value in required): + raise ValueError('An unread mark needs a conversation id, a user id, a message id and a planned time.') + attempts = max(1, int(max_retries) + 1) + conversation_item = None + for _attempt in range(attempts): + try: + conversation_item = container.read_item(item=conversation_id, partition_key=conversation_id) + except exceptions.CosmosResourceNotFoundError: + return UNREAD_NOT_FOUND, None + if _conversation_unavailable(conversation_item, user_id, require_private): + return UNREAD_UNAVAILABLE, conversation_item + if conversation_item.get('last_unread_assistant_message_id') == message_id: + return UNREAD_ALREADY_MARKED, conversation_item + if skip_if_read_since is not None and _reached(conversation_item.get('last_updated'), skip_if_read_since): + return UNREAD_READ_SINCE, conversation_item + + candidate = dict(conversation_item) + if not _reached(candidate.get('last_updated'), planned_at): + candidate['last_updated'] = planned_at + candidate = mark_conversation_unread(candidate, message_id, planned_at) + try: + stored = container.replace_item( + item=conversation_id, + body=candidate, + etag=conversation_item.get('_etag'), + match_condition=MatchConditions.IfNotModified, + ) + except exceptions.CosmosResourceNotFoundError: + return UNREAD_NOT_FOUND, None + except exceptions.CosmosHttpResponseError as exc: + if exc.status_code != 412: + raise + continue + return UNREAD_MARKED, stored if isinstance(stored, dict) else candidate + return UNREAD_CONFLICT, conversation_item diff --git a/application/single_app/functions_notifications.py b/application/single_app/functions_notifications.py index a39a90bb8..d445e6a89 100644 --- a/application/single_app/functions_notifications.py +++ b/application/single_app/functions_notifications.py @@ -34,6 +34,9 @@ KEY_VAULT_SECRET_REMINDER_NOTIFICATION_TYPE = 'key_vault_secret_expiring' M365_APPROVAL_PENDING_NOTIFICATION_TYPE = 'm365_approval_pending' M365_APPROVAL_UPDATED_NOTIFICATION_TYPE = 'm365_approval_updated' +# Pinned by a parity test to functions_workflow_chat_delivery.NOTIFICATION_TYPE, which this module +# does not import so its import graph stays unchanged. +WORKFLOW_CHAT_DELIVERY_NOTIFICATION_TYPE = 'workflow_chat_delivery' MAX_NOTIFICATION_IDEMPOTENCY_KEY_LENGTH = 512 WORKFLOW_ALERT_PRIORITY_CONFIG = { 'info': { @@ -209,6 +212,10 @@ 'icon': 'bi-bell', 'color': 'secondary' }, + WORKFLOW_CHAT_DELIVERY_NOTIFICATION_TYPE: { + 'icon': 'bi-activity', + 'color': 'info' + }, KEY_VAULT_SECRET_REMINDER_NOTIFICATION_TYPE: { 'icon': 'bi-safe', 'color': 'warning' @@ -715,8 +722,15 @@ def create_chat_response_notification( message_id, conversation_title='', response_preview='', + *, + idempotency_key=None, + strict=False, ): - """Create a personal notification when a chat response completes.""" + """Create a personal notification when a chat response completes. + + ``idempotency_key`` and ``strict`` pass through to ``create_notification``, so a server + retry with the same key returns the existing notice rather than creating a second one. + """ normalized_title = str(conversation_title or '').strip() or 'Conversation' normalized_preview = str(response_preview or '').strip() if len(normalized_preview) > 160: @@ -740,7 +754,9 @@ def create_chat_response_notification( metadata={ 'conversation_id': conversation_id, 'message_id': message_id, - } + }, + idempotency_key=idempotency_key, + strict=strict, ) diff --git a/application/single_app/functions_orchestration_workflow_runs.py b/application/single_app/functions_orchestration_workflow_runs.py index 7d2dfbfb7..c63c35391 100644 --- a/application/single_app/functions_orchestration_workflow_runs.py +++ b/application/single_app/functions_orchestration_workflow_runs.py @@ -73,6 +73,14 @@ workflow_run_ready, workflow_run_settings_gate, ) +from functions_workflow_chat_delivery import ( + WORKFLOW_RUN_DELIVERY_FOLLOW_UP, + WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY, + build_chat_delivery_seed, + chat_delivery_applies, + normalize_model_selection, + normalize_requester_roles, +) WORKFLOW_RUN_OUTPUT = 'run' @@ -200,8 +208,9 @@ } _DEFAULT_WORKFLOW_NAME = 'Workflow' -# The reply's note about the saved workflows a plan started. It says where a run's results appear -# and never promises them in the chat, which only a later delivery step can do. +# The reply's note about the saved workflows a plan started. It says where a run's results appear. +# It promises them in the chat only when every run it started recorded a chat delivery, because only +# then does the server post each run's result back into this conversation when the run finishes. WORKFLOW_RUN_NOTE_HEADING = 'Saved workflows:' WORKFLOW_RUN_FOLLOW_UP = ( "Follow the run's progress and results in the workflow's run history in Workflows. Results also " @@ -585,6 +594,40 @@ def _started_status(run): return WORKFLOW_RUN_STATUS_QUEUED +def _chat_delivery_seed(settings, context, planning): + """Return the record that has the run's result posted back to this chat, or None. + + The cheap settings check comes first, so a deployment with Use Workflow Results In Chat off + never reaches the role-aware gate. A failure here only means the run is not delivered: it + never fails the start. + """ + if not isinstance(settings, dict) or not settings.get('enable_chat_workflow_results'): + return None + try: + # Settings initialize application storage, so they are imported only once a gate is reached. + from functions_settings import is_chat_workflow_results_enabled_for_user + + user_roles = getattr(context, 'user_roles', None) + if not is_chat_workflow_results_enabled_for_user(settings, user_roles=user_roles): + return None + time_zone = getattr(context, 'time_zone', None) + if not time_zone and isinstance(planning, dict): + time_zone = planning.get('time_zone') + return build_chat_delivery_seed( + time_zone=time_zone, + model_selection=normalize_model_selection( + getattr(context, 'seeds', None), getattr(context, 'active_group_ids', None), + ), + requester_roles=normalize_requester_roles(user_roles), + ) + except Exception as exc: + _log( + 'Chat delivery was not recorded for a started workflow run.', logging.WARNING, + run_id=getattr(context, 'run_id', None), error_type=type(exc).__name__, + ) + return None + + def _start(step, context, *, settings, user_id, recheck, requested_at): """Start or link the step's workflow run. Returns an outcome; raises only to fail the step. @@ -651,6 +694,7 @@ def unavailable(reason, workflow_id=None): return unavailable(RULE_NOT_DURABLE, workflow_id) if workflow.get('status') in M365_ACTIVE_STATES: return unavailable(REASON_WAITING_FOR_MICROSOFT_365, workflow_id) + chat_delivery = _chat_delivery_seed(settings, context, planning) recheck() runtime_unavailable, runtime_conflict = _runtime_errors() @@ -671,6 +715,7 @@ def unavailable(reason, workflow_id=None): 'requested_by': user_id, 'requested_at': requested_at, }, + **({'chat_delivery': chat_delivery} if chat_delivery is not None else {}), ) except runtime_conflict as exc: code = getattr(exc, 'code', None) @@ -684,11 +729,19 @@ def unavailable(reason, workflow_id=None): except AzureError as exc: raise _WorkflowRunStepFailure(FAILURE_RUNTIME_UNAVAILABLE) from exc run = queued.get('run') if isinstance(queued, dict) else None - return _outcome(_started_status(run), name=name, workflow_id=workflow_id, run_id=run_id, queued=True) + outcome = _outcome(_started_status(run), name=name, workflow_id=workflow_id, run_id=run_id, queued=True) + # Only a run this call started, carrying this chat's delivery record, is posted back here. + if ( + chat_delivery is not None + and outcome['status'] in (WORKFLOW_RUN_STATUS_QUEUED, WORKFLOW_RUN_STATUS_RUNNING) + and chat_delivery_applies(run, context.conversation_id) + ): + outcome['chat_delivery'] = True + return outcome def _sidecar(step, context, *, user_id, producer_run_id, outcome): - return { + sidecar = { 'version': WORKFLOW_RUN_VERSION, 'step_id': step['step_id'], 'orchestration_run_id': producer_run_id, @@ -702,6 +755,10 @@ def _sidecar(step, context, *, user_id, producer_run_id, outcome): 'status': outcome.get('status'), 'reason': outcome.get('reason'), } + # Set only by a start that recorded this chat's delivery, never for a linked or rebuilt run. + if outcome.get('chat_delivery') is True: + sidecar['chat_delivery'] = True + return sidecar def _failure(exc): @@ -879,7 +936,7 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): record['step_id']: record for record in (execution_steps if isinstance(execution_steps, list) else ()) if isinstance(record, dict) and isinstance(record.get('step_id'), str) } - lines, started = [], 0 + lines, started, delivered = [], 0, 0 for step in steps: if not isinstance(step, dict) or step.get('capability_id') != CAPABILITY_WORKFLOW_RUN: continue @@ -896,6 +953,8 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): continue if sidecar['status'] in WORKFLOW_RUN_STARTED_STATUSES: started += 1 + if sidecar.get('chat_delivery') is True: + delivered += 1 lines.append(_step_line(sidecar)) notes = plan.get('workflow_run_notes') if isinstance(plan.get('workflow_run_notes'), list) else [] for note in notes: @@ -909,7 +968,11 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): return '' parts = ['\n'.join([WORKFLOW_RUN_NOTE_HEADING, *lines])] if started: - parts.append(WORKFLOW_RUN_FOLLOW_UP_MANY if started > 1 else WORKFLOW_RUN_FOLLOW_UP) + # The chat promises the result only when every run started here will be posted back to it. + if delivered == started: + parts.append(WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY if started > 1 else WORKFLOW_RUN_DELIVERY_FOLLOW_UP) + else: + parts.append(WORKFLOW_RUN_FOLLOW_UP_MANY if started > 1 else WORKFLOW_RUN_FOLLOW_UP) if stopped: parts.append(WORKFLOW_RUN_STOPPED) return '\n\n'.join(parts) @@ -919,6 +982,8 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): 'FAILURE_RUNTIME_UNAVAILABLE', 'NO_WORKFLOW_STARTED', 'REASON_INVALID', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY', 'WORKFLOW_RUN_FOLLOW_UP', 'WORKFLOW_RUN_FOLLOW_UP_MANY', 'WORKFLOW_RUN_NOTE_HEADING', diff --git a/application/single_app/functions_personal_workflows.py b/application/single_app/functions_personal_workflows.py index 589ca9b2a..1b9aa57ea 100644 --- a/application/single_app/functions_personal_workflows.py +++ b/application/single_app/functions_personal_workflows.py @@ -8,6 +8,7 @@ import uuid from datetime import datetime, timedelta, timezone +from azure.core import MatchConditions from azure.cosmos import exceptions from config import ( @@ -40,6 +41,7 @@ from functions_settings import get_settings, get_user_settings, normalize_model_endpoints from functions_workflow_alerts import normalize_workflow_alert_settings from functions_workflow_alert_safety import sanitize_workflow_alert_record +from functions_workflow_chat_delivery import delivery_log, merge_stored_run_fields, needs_guarded_run_save from functions_workflow_result_store import delete_workflow_run_results from functions_workflow_bindings import authorize_workflow_reference from functions_workflow_definition_store import ( @@ -86,6 +88,8 @@ WORKFLOW_TASK_NAME_MAX_LENGTH = 120 WORKFLOW_TASK_RUNNER_TYPES = {'inherit', 'agent', 'model'} WORKFLOW_CONVERSATION_ACCESS_ERROR = 'Workflow conversation not found or access denied.' +# A chat-started run's guarded save re-reads and retries at most this many times on a conflict. +_GUARDED_RUN_SAVE_ATTEMPTS = 8 def _utc_now(): return datetime.now(timezone.utc) @@ -1274,10 +1278,52 @@ def save_personal_workflow_run(user_id, run_record): run_record = run_record if isinstance(run_record, dict) else {} run_record['user_id'] = user_id run_record.setdefault('id', str(uuid.uuid4())) + if needs_guarded_run_save(run_record): + return _strip_cosmos_metadata(_save_chat_started_workflow_run(user_id, run_record)) result = cosmos_personal_workflow_runs_container.upsert_item(body=run_record) return _strip_cosmos_metadata(result) +def _save_chat_started_workflow_run(user_id, run_record): + """Save a run a chat plan started without undoing what another writer stored meanwhile. + + The stored chat delivery record, chat invocation and cancellation request win over this copy, + and each replace lands only on the stored document it was merged from. A save that cannot be + guarded falls back to an upsert of the best merge it has, which the delivery's own message id, + notification key and unread rules keep from posting or announcing a result twice. + """ + merged = run_record + for _attempt in range(_GUARDED_RUN_SAVE_ATTEMPTS): + try: + stored = cosmos_personal_workflow_runs_container.read_item( + item=run_record['id'], partition_key=user_id, + ) + except exceptions.CosmosResourceNotFoundError: + return cosmos_personal_workflow_runs_container.upsert_item(body=run_record) + except Exception as exc: + delivery_log( + 'A chat-started workflow run was saved without its guard.', level=logging.WARNING, + run_id=run_record.get('id'), reason='read_failed', error_type=type(exc).__name__, + ) + return cosmos_personal_workflow_runs_container.upsert_item(body=merged) + merged = merge_stored_run_fields(run_record, stored) + try: + return cosmos_personal_workflow_runs_container.replace_item( + item=run_record['id'], body=merged, etag=stored.get('_etag'), + match_condition=MatchConditions.IfNotModified, + ) + except exceptions.CosmosResourceNotFoundError: + return cosmos_personal_workflow_runs_container.upsert_item(body=merged) + except exceptions.CosmosHttpResponseError as exc: + if exc.status_code != 412: + raise + delivery_log( + 'A chat-started workflow run was saved without its guard.', level=logging.WARNING, + run_id=run_record.get('id'), reason='conflicts_exhausted', + ) + return cosmos_personal_workflow_runs_container.upsert_item(body=merged) + + def save_personal_workflow_run_item(user_id, item_record): """Create or update a per-item workflow run record.""" item_record = item_record if isinstance(item_record, dict) else {} diff --git a/application/single_app/functions_workflow_chat_delivery.py b/application/single_app/functions_workflow_chat_delivery.py new file mode 100644 index 000000000..6b9ed2f33 --- /dev/null +++ b/application/single_app/functions_workflow_chat_delivery.py @@ -0,0 +1,992 @@ +# functions_workflow_chat_delivery.py +"""Pure chat delivery contract for chat-started workflow runs. + +Version: 0.261.227 +Implemented in: 0.261.227 + +This module owns the pure contract for posting a chat-started workflow run's result back into the +chat. It performs no storage, model calls, or result-text reads. Inline copies of ``_fingerprint``, +``clean_catalog_text``, ``quoted_workflow_name``, state sets, and model identity fields are pinned +by parity tests because importing their sources would create import cycles or pull azure.cosmos or +config into this module. +""" + +import hashlib +import json +import logging +import re +import threading +import uuid +from collections import deque +from collections.abc import Mapping +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from urllib.parse import urlencode + +from functions_appinsights import log_event, workflow_log_context +from functions_m365_workflow_binding import M365_WAITING_STATES + + +CHAT_DELIVERY_VERSION = 1 +CHAT_DELIVERY_KEY = 'chat_delivery' +DELIVERY_METADATA_KEY = 'workflow_delivery' +DELIVERY_MESSAGE_ID_PREFIX = 'assistant_workflow_delivery_' +NOTIFICATION_TYPE = 'workflow_chat_delivery' +WORKFLOW_SCOPE = 'personal' +CHAT_TRIGGER_SOURCE = 'chat_orchestration' +LOG_PREFIX = '[WORKFLOW_CHAT_DELIVERY]' +STATUS_LOG_PREFIX = '[WORKFLOW_CHAT_DELIVERY_STATUS]' + +STATUS_PENDING = 'pending' +STATUS_READY = 'ready' +STATUS_DELIVERING = 'delivering' +STATUS_DELIVERED = 'delivered' +STATUS_UNDELIVERABLE = 'undeliverable' +STATUS_EXPIRED = 'expired' +STATUSES = frozenset({ + STATUS_PENDING, + STATUS_READY, + STATUS_DELIVERING, + STATUS_DELIVERED, + STATUS_UNDELIVERABLE, + STATUS_EXPIRED, +}) +OPEN_STATUSES = frozenset({STATUS_PENDING, STATUS_READY, STATUS_DELIVERING}) +FINAL_STATUSES = frozenset({STATUS_DELIVERED, STATUS_UNDELIVERABLE, STATUS_EXPIRED}) +REOPENABLE_STATUSES = frozenset({STATUS_DELIVERED, STATUS_UNDELIVERABLE}) + +KIND_RESULT = 'result' +KIND_FAILED = 'failed' +KIND_CANCELLED = 'cancelled' +KIND_STATUS = 'status' +KIND_CONTENT_BLOCKED = 'content_blocked' +KIND_ANALYSIS = 'analysis' +KIND_EXPIRED = 'expired' +KIND_SKIPPED = 'skipped' +RESULT_CONTEXT_KINDS = frozenset({KIND_RESULT, KIND_ANALYSIS}) + +PHASE_CLAIMED = 'claimed' +PHASE_PUBLISHING = 'publishing' +PHASE_UNREAD_MARKED = 'unread_marked' +PHASE_MESSAGE_CREATED = 'message_created' +PHASE_NOTIFIED = 'notified' +PHASES = (PHASE_CLAIMED, PHASE_PUBLISHING, PHASE_UNREAD_MARKED, PHASE_MESSAGE_CREATED, PHASE_NOTIFIED) + +NOTICE_CHAT_RESPONSE = 'chat_response' +NOTICE_UNDELIVERABLE = 'undeliverable' +NOTICE_EXPIRED = 'expired' +NOTICE_NONE = 'none' +NOTICE_KINDS = frozenset({NOTICE_CHAT_RESPONSE, NOTICE_UNDELIVERABLE, NOTICE_EXPIRED, NOTICE_NONE}) + +REASON_CHAT_UNAVAILABLE = 'chat_unavailable' +REASON_ACCESS_LOST = 'access_lost' +REASON_WORKFLOW_DELETED = 'workflow_deleted' +REASON_RUNTIME_MISSING = 'runtime_missing' +REASON_DELIVERY_FAILED = 'delivery_failed' +REASON_EXPIRED_BEFORE_DELIVERY = 'expired_before_delivery' +REASON_CONTENT_BLOCKED = 'content_blocked' +REASON_RESULTS_OFF = 'results_off' +REASON_RESULT_UNAVAILABLE = 'result_unavailable' +REASON_DEADLINE_EXCEEDED = 'deadline_exceeded' +SILENT_REASONS = frozenset({REASON_WORKFLOW_DELETED, REASON_RUNTIME_MISSING}) + +RUNTIME_TERMINAL_STATES = frozenset({ + 'completed', + 'completed_partial', + 'failed', + 'invalid', + 'incomplete', + 'cancelled', + 'skipped', +}) +RESULT_RUN_STATES = frozenset({'completed', 'completed_partial'}) +FAILED_RUN_STATES = frozenset({'failed', 'invalid', 'incomplete'}) +RUNTIME_RESUMABLE_STATES = frozenset({'failed', 'invalid', 'incomplete'}) +RUNTIME_WAITING_STATES = frozenset({'waiting_approval', 'waiting_output', 'waiting_recovery', 'paused'}) | M365_WAITING_STATES +RUNTIME_ACTIVE_STATES = frozenset({'queued', 'running', 'cancelling'}) +RUN_DOCUMENT_TERMINAL_STATUSES = RUNTIME_TERMINAL_STATES | frozenset({'canceled'}) + +KIND_BY_TERMINAL_STATE = { + 'completed': KIND_RESULT, + 'completed_partial': KIND_RESULT, + 'failed': KIND_FAILED, + 'invalid': KIND_FAILED, + 'incomplete': KIND_FAILED, + 'cancelled': KIND_CANCELLED, + 'skipped': KIND_SKIPPED, +} +MODEL_IDENTITY_FIELDS = ('model_deployment', 'model_id', 'model_endpoint_id', 'model_provider') + +DELIVERY_GRACE_SECONDS = 86400 +DEFAULT_DEADLINE_SECONDS = 86400 +MAX_DEADLINE_SECONDS = 86400 +LEASE_SECONDS = 300 +SWEEP_INTERVAL_SECONDS = 30 +SWEEP_LOCK_NAME = 'workflow_chat_delivery_sweep' +SWEEP_LOCK_SECONDS = 120 +SWEEP_TOP = 25 +SWEEP_MAX_RUNS = 8 +SWEEP_MAX_SECONDS = 90 +SWEEP_TS_GRACE_SECONDS = 120 +BACKOFF_SECONDS = (30, 60, 120, 240, 480, 900, 900, 900) +MAX_ATTEMPTS = 8 +DEFER_RETRY_SECONDS = 30 +DEFER_MAX_SECONDS = 900 +STREAM_META_FRESH_SECONDS = 90 +RECENT_USER_MESSAGE_SECONDS = 600 + +HINT_QUEUE_MAX = 256 +HISTORY_MAX = 5 +MAX_ROLES = 20 +MAX_ROLE_LENGTH = 64 +MAX_GROUPS = 10 +MAX_GROUP_ID_LENGTH = 64 +MAX_MODEL_FIELD_LENGTH = 200 +MAX_REASONING_EFFORT_LENGTH = 32 +NAME_MAX_LENGTH = 80 + +COMPOSE_TEMPERATURE = 0.3 +COMPOSE_MAX_TOKENS = 1500 +REPLY_MAX_CHARS = 12000 +TRIMMED_FALLBACK_CHARS = 2000 +EXCERPT_BUDGET_BYTES = 48 * 1024 +TRIMMED_FALLBACK_INTRO = "I couldn't summarize the result, so here's the start of it:" +FALLBACK_QUESTION = 'Summarize what this workflow run produced.' +WORKFLOW_RUN_DELIVERY_FOLLOW_UP = ( + "I'll post the results here when the run finishes. You can also follow it in the workflow's " + "run history in Workflows." +) +WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY = ( + "I'll post each run's results here when it finishes. You can also follow them in each " + "workflow's run history in Workflows." +) +UNDELIVERABLE_NOTICE_MESSAGE = "The chat that started this run can't show it anymore. Open the run in Workflows to see the details." +EXPIRED_NOTICE_MESSAGE = "The run didn't finish in time to post to the chat. Open it in Workflows to see where it stands." +DELIVERY_RETRY_UNSUPPORTED = 'workflow_delivery_retry_unsupported' +DELIVERY_EDIT_UNSUPPORTED = 'workflow_delivery_edit_unsupported' +_DELIVERY_REFUSALS = { + DELIVERY_RETRY_UNSUPPORTED: ( + "A workflow run posted this message, so it can't be retried here. " + "To run the workflow again, open the run in Workflows." + ), + DELIVERY_EDIT_UNSUPPORTED: "A workflow run posted this message, so it can't be edited. Ask a new question instead.", +} +_DEFAULT_WORKFLOW_NAME = 'Workflow' +_TEXT_NOISE = re.compile('[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff]') +_TIME_ZONE_RE = re.compile(r'[A-Za-z0-9_+\-/]{1,64}') + +_HINT_LOCK = threading.Lock() +_HINT_EVENT = threading.Event() +_HINT_QUEUE = deque() +_HINT_KEYS = set() + + +def _fingerprint(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode('utf-8') + ).hexdigest() + + +def clean_catalog_text(value, limit): + """Return user-authored text as one bounded, printable line, or '' for anything else.""" + if not isinstance(value, str): + return '' + text = ' '.join(_TEXT_NOISE.sub(' ', value).split()) + if len(text) <= limit: + return text + return text[:limit - 1].rstrip() + '\u2026' + + +def quoted_workflow_name(value): + """A saved name as inline code that cannot format the reply, or None when nothing is left of it.""" + text = clean_catalog_text(value, NAME_MAX_LENGTH) + if not text.replace('`', '').strip(): + return None + return '`' + text.replace('`', "'") + '`' + + +def display_workflow_name(value): + return quoted_workflow_name(value) or f'`{_DEFAULT_WORKFLOW_NAME}`' + + +def utc_now(): + return datetime.now(timezone.utc) + + +def format_delivery_timestamp(dt): + if not isinstance(dt, datetime): + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + else: + dt = dt.astimezone(timezone.utc) + return dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +def parse_delivery_timestamp(value): + if isinstance(value, datetime): + dt = value + elif isinstance(value, str): + text = value.strip() + if not text or len(text) > 64: + return None + if text.endswith('Z'): + text = text[:-1] + '+00:00' + try: + dt = datetime.fromisoformat(text) + except (TypeError, ValueError): + return None + else: + return None + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def workflow_delivery_message_id(run_id, conversation_id, generation): + if not isinstance(run_id, str) or not run_id.strip(): + raise ValueError('run_id is required') + if not isinstance(conversation_id, str) or not conversation_id.strip(): + raise ValueError('conversation_id is required') + if type(generation) is not int or generation < 0: + raise ValueError('generation must be a non-negative integer') + return DELIVERY_MESSAGE_ID_PREFIX + _fingerprint([run_id, conversation_id, generation])[:40] + + +def delivery_thread_id(message_id): + if not isinstance(message_id, str) or not message_id.strip(): + raise ValueError('message_id is required') + return str(uuid.uuid5(uuid.NAMESPACE_URL, message_id)) + + +def is_workflow_delivery_message(message): + if not isinstance(message, Mapping): + return False + message_id = message.get('id') + if isinstance(message_id, str) and message_id.startswith(DELIVERY_MESSAGE_ID_PREFIX): + return True + metadata = message.get('metadata') + return isinstance(metadata, Mapping) and isinstance(metadata.get(DELIVERY_METADATA_KEY), Mapping) + + +def workflow_delivery_refusal_payload(code): + """Return the fixed 400 body for Retry or Edit on a message a workflow run posted.""" + if code not in _DELIVERY_REFUSALS: + code = DELIVERY_RETRY_UNSUPPORTED + return {'error': _DELIVERY_REFUSALS[code], 'code': code}, 400 + + +def normalize_requester_roles(roles): + if not isinstance(roles, (list, tuple)): + return [] + normalized = [] + seen = set() + for role in roles: + if not isinstance(role, str) or not role.strip() or len(role) > MAX_ROLE_LENGTH or role in seen: + continue + normalized.append(role) + seen.add(role) + if len(normalized) >= MAX_ROLES: + break + return normalized + + +def normalize_model_selection(seeds, active_group_ids=None): + if not isinstance(seeds, Mapping): + seeds = {} + model = {} + raw_model = seeds.get('model') + if isinstance(raw_model, Mapping): + invalid = False + for field in MODEL_IDENTITY_FIELDS: + value = raw_model.get(field) + if value is None or value == '': + continue + if not isinstance(value, str): + invalid = True + break + text = value.strip() + if not text or len(text) > MAX_MODEL_FIELD_LENGTH: + invalid = True + break + model[field] = text + if invalid: + model = {} + reasoning_effort = seeds.get('reasoning_effort') + if isinstance(reasoning_effort, str): + reasoning_effort = reasoning_effort.strip() + if len(reasoning_effort) > MAX_REASONING_EFFORT_LENGTH: + reasoning_effort = '' + else: + reasoning_effort = '' + groups_source = seeds.get('active_group_ids') + if not isinstance(groups_source, (list, tuple)) or not groups_source: + groups_source = active_group_ids + active_groups = [] + seen_groups = set() + if isinstance(groups_source, (list, tuple)): + for group_id in groups_source: + if not isinstance(group_id, str): + continue + text = group_id.strip() + if not text or len(text) > MAX_GROUP_ID_LENGTH or text in seen_groups: + continue + active_groups.append(text) + seen_groups.add(text) + if len(active_groups) >= MAX_GROUPS: + break + return {'model': model, 'reasoning_effort': reasoning_effort, 'active_group_ids': active_groups} + + +def normalize_time_zone(value): + if isinstance(value, str) and _TIME_ZONE_RE.fullmatch(value): + return value + return 'UTC' + + +def build_chat_delivery_seed(*, time_zone, model_selection, requester_roles, now=None): + created_at = format_delivery_timestamp(now if isinstance(now, datetime) else utc_now()) + return { + 'version': CHAT_DELIVERY_VERSION, + 'status': STATUS_PENDING, + 'generation': None, + 'expires_at': None, + 'time_zone': normalize_time_zone(time_zone), + 'model_selection': normalize_model_selection(model_selection), + 'requester_roles': normalize_requester_roles(requester_roles), + 'lease_id': None, + 'lease_expires_at': None, + 'attempts': 0, + 'next_attempt_at': None, + 'first_deferred_at': None, + 'phase': None, + 'message_id': None, + 'planned_at': None, + 'kind': None, + 'notice_kind': None, + 'outcome_reason': None, + 'run_status': None, + 'delivered_at': None, + 'created_at': created_at, + 'updated_at': created_at, + 'history': [], + } + + +def _lower_text(value): + if not isinstance(value, str): + return None + text = value.strip().lower() + return text or None + + +def control_summary(control): + if not isinstance(control, Mapping): + return { + 'exists': False, + 'deleted': False, + 'version': None, + 'state': None, + 'phase': None, + 'gate_reason_code': None, + 'deadline_at': None, + 'deadline_seconds': None, + 'schema_version': None, + } + limits = control.get('limits') if isinstance(control.get('limits'), Mapping) else {} + gate = control.get('gate') if isinstance(control.get('gate'), Mapping) else {} + version = control.get('version') + schema_version = control.get('schema_version') + return { + 'exists': True, + 'deleted': bool(control.get('deleted')), + 'version': version if type(version) is int else None, + 'state': _lower_text(control.get('state') if control.get('state') is not None else control.get('control_state')), + 'phase': _lower_text(control.get('phase')), + 'gate_reason_code': _lower_text(gate.get('reason_code')), + 'deadline_at': control.get('deadline_at') if control.get('deadline_at') is not None else limits.get('deadline_at'), + 'deadline_seconds': control.get('deadline_seconds') if control.get('deadline_seconds') is not None else limits.get('deadline_seconds'), + 'schema_version': schema_version if type(schema_version) is int else None, + } + + +def compute_expires_at(*, deadline_at=None, deadline_seconds=None, base=None): + parsed_deadline = parse_delivery_timestamp(deadline_at) + if parsed_deadline: + return format_delivery_timestamp(parsed_deadline + timedelta(seconds=DELIVERY_GRACE_SECONDS)) + parsed_base = parse_delivery_timestamp(base) or utc_now() + seconds = DEFAULT_DEADLINE_SECONDS + if type(deadline_seconds) is int and deadline_seconds > 0: + seconds = min(deadline_seconds, MAX_DEADLINE_SECONDS) + return format_delivery_timestamp(parsed_base + timedelta(seconds=seconds + DELIVERY_GRACE_SECONDS)) + + +def finalize_chat_delivery_seed(seed, control): + if not isinstance(seed, Mapping): + return None + record = deepcopy(dict(seed)) + summary = control_summary(control) + record['expires_at'] = compute_expires_at( + deadline_at=summary.get('deadline_at'), + deadline_seconds=summary.get('deadline_seconds'), + base=record.get('created_at'), + ) + return record + + +def chat_delivery_applies(run, conversation_id): + if not isinstance(run, Mapping) or not isinstance(conversation_id, str) or not conversation_id: + return False + delivery = run.get(CHAT_DELIVERY_KEY) + invocation = run.get('chat_invocation') + return ( + isinstance(delivery, Mapping) + and type(delivery.get('version')) is int + and delivery.get('version') == CHAT_DELIVERY_VERSION + and isinstance(invocation, Mapping) + and invocation.get('conversation_id') == conversation_id + ) + + +def needs_guarded_run_save(run): + if not isinstance(run, Mapping): + return False + return CHAT_DELIVERY_KEY in run or (run.get('trigger_source') == CHAT_TRIGGER_SOURCE and 'chat_invocation' not in run) + + +def merge_stored_run_fields(incoming, stored): + incoming = incoming if isinstance(incoming, Mapping) else {} + stored = stored if isinstance(stored, Mapping) else {} + cosmos_keys = {'_rid', '_self', '_etag', '_attachments', '_ts'} + merged = {key: deepcopy(value) for key, value in incoming.items() if key not in cosmos_keys} + if CHAT_DELIVERY_KEY in stored: + merged[CHAT_DELIVERY_KEY] = deepcopy(stored.get(CHAT_DELIVERY_KEY)) + else: + merged.pop(CHAT_DELIVERY_KEY, None) + if 'chat_invocation' not in merged and 'chat_invocation' in stored: + merged['chat_invocation'] = deepcopy(stored.get('chat_invocation')) + stored_status = _lower_text(stored.get('status')) + has_cancel_request = bool(stored.get('cancellation_requested_at')) or stored_status in {'cancelling', 'cancelled', 'canceled'} + if has_cancel_request: + for field in ('cancellation_requested_at', 'cancellation_requested_by'): + if stored.get(field): + merged[field] = deepcopy(stored.get(field)) + incoming_status = _lower_text(merged.get('status')) + if stored_status in {'cancelled', 'canceled'}: + merged['status'] = 'cancelled' + elif incoming_status not in {'cancelled', 'canceled'}: + merged['status'] = 'cancelling' + return merged + + +def _mark_closed(record, reason, timestamp): + updated = deepcopy(dict(record)) + updated.update({ + 'status': STATUS_UNDELIVERABLE, + 'outcome_reason': reason, + 'notice_kind': NOTICE_NONE, + 'lease_id': None, + 'lease_expires_at': None, + 'next_attempt_at': None, + 'updated_at': timestamp, + }) + return updated + + +def _mark_delivered_without_notice(record, timestamp): + updated = deepcopy(dict(record)) + updated.update({ + 'status': STATUS_DELIVERED, + 'notice_kind': NOTICE_NONE, + 'delivered_at': updated.get('delivered_at') or timestamp, + 'lease_id': None, + 'lease_expires_at': None, + 'next_attempt_at': None, + 'updated_at': timestamp, + }) + return updated + + +def _close_for_runtime(record, reason, timestamp): + # A message that already exists stays delivered; only an unposted generation closes. + if phase_at_least(record.get('phase'), PHASE_MESSAGE_CREATED): + return _mark_delivered_without_notice(record, timestamp) + return _mark_closed(record, reason, timestamp) + + +def _abandon_generation(current, candidate): + """Reset the per-generation fields when a ready generation is replaced before it finished.""" + updated = deepcopy(candidate) + if phase_at_least(current.get('phase'), PHASE_MESSAGE_CREATED): + history = list(current.get('history') if isinstance(current.get('history'), list) else []) + entry = _history_entry(current) + entry.update({'status': STATUS_DELIVERED, 'notice_kind': NOTICE_NONE}) + history.append(entry) + updated['history'] = history[-HISTORY_MAX:] + updated.update({ + 'phase': None, + 'planned_at': None, + 'message_id': None, + 'attempts': 0, + 'outcome_reason': None, + 'notice_kind': None, + 'delivered_at': None, + 'next_attempt_at': None, + 'first_deferred_at': None, + 'lease_id': None, + 'lease_expires_at': None, + }) + return updated + + +def _generation_from_summary(summary): + version = summary.get('version') if isinstance(summary, Mapping) else None + if type(version) is int and version >= 0: + return version + return 0 + + +def _ready_fields(summary, kind): + state = summary.get('state') if isinstance(summary, Mapping) else None + return { + 'status': STATUS_READY, + 'generation': _generation_from_summary(summary), + 'kind': kind, + 'run_status': state, + } + + +def _history_entry(record): + return { + 'generation': record.get('generation'), + 'status': record.get('status'), + 'kind': record.get('kind'), + 'outcome_reason': record.get('outcome_reason'), + 'message_id': record.get('message_id'), + 'delivered_at': record.get('delivered_at'), + 'notice_kind': record.get('notice_kind'), + } + + +def _changed_record(original, candidate): + original_without_updated = {key: value for key, value in original.items() if key != 'updated_at'} + candidate_without_updated = {key: value for key, value in candidate.items() if key != 'updated_at'} + return original_without_updated != candidate_without_updated + + +def reconcile_chat_delivery(record, summary, now=None): + timestamp_dt = now if isinstance(now, datetime) else utc_now() + timestamp = format_delivery_timestamp(timestamp_dt) + if not isinstance(record, Mapping): + return deepcopy(record), False, False + current = deepcopy(dict(record)) + if current.get('version') != CHAT_DELIVERY_VERSION or current.get('status') not in STATUSES: + return current, False, False + status = current.get('status') + if status in {STATUS_DELIVERING, STATUS_EXPIRED}: + return current, False, False + if not isinstance(summary, Mapping) or not summary.get('exists'): + if status in {STATUS_PENDING, STATUS_READY}: + updated = _close_for_runtime(current, REASON_RUNTIME_MISSING, timestamp) + return updated, True, False + return current, False, False + if summary.get('deleted'): + if status in {STATUS_PENDING, STATUS_READY}: + updated = _close_for_runtime(current, REASON_WORKFLOW_DELETED, timestamp) + return updated, True, False + return current, False, False + reopened = False + if status in REOPENABLE_STATUSES and type(current.get('generation')) is int: + version = summary.get('version') + if type(version) is int and version > current.get('generation'): + history = list(current.get('history') if isinstance(current.get('history'), list) else []) + history.append(_history_entry(current)) + current.update({ + 'status': STATUS_PENDING, + 'generation': None, + 'kind': None, + 'notice_kind': None, + 'outcome_reason': None, + 'run_status': None, + 'delivered_at': None, + 'message_id': None, + 'planned_at': None, + 'phase': None, + 'lease_id': None, + 'lease_expires_at': None, + 'next_attempt_at': None, + 'first_deferred_at': None, + 'attempts': 0, + 'expires_at': compute_expires_at( + deadline_at=summary.get('deadline_at'), + deadline_seconds=summary.get('deadline_seconds'), + base=timestamp_dt, + ), + 'history': history[-HISTORY_MAX:], + }) + status = STATUS_PENDING + reopened = True + if status in {STATUS_PENDING, STATUS_READY}: + state = summary.get('state') + candidate = deepcopy(current) + expires_at = parse_delivery_timestamp(candidate.get('expires_at')) + if state in RUNTIME_TERMINAL_STATES: + candidate.update(_ready_fields(summary, KIND_BY_TERMINAL_STATE[state])) + elif ( + state == 'paused' + and REASON_DEADLINE_EXCEEDED in {summary.get('gate_reason_code'), summary.get('phase')} + ) or (state not in RUNTIME_TERMINAL_STATES and expires_at and timestamp_dt > expires_at): + candidate.update(_ready_fields(summary, KIND_EXPIRED)) + elif status == STATUS_READY: + candidate.update({'status': STATUS_PENDING, 'generation': None, 'kind': None, 'run_status': None}) + if status == STATUS_READY and candidate.get('generation') != current.get('generation'): + # The run moved on (resumed, or resumed and finished again) before this generation was posted. + candidate = _abandon_generation(current, candidate) + if candidate.get('status') == STATUS_READY and ( + status == STATUS_PENDING or candidate.get('generation') != current.get('generation') + ): + # A record that just became deliverable is due now, not after a wait scheduled while it was pending. + candidate.update({'next_attempt_at': None, 'first_deferred_at': None}) + if reopened or _changed_record(current, candidate): + candidate['updated_at'] = timestamp + return candidate, True, candidate.get('status') == STATUS_READY + return current, False, current.get('status') == STATUS_READY + return current, False, False + + +def phase_at_least(phase, wanted): + if phase not in PHASES or wanted not in PHASES: + return False + return PHASES.index(phase) >= PHASES.index(wanted) + + +def delivery_label(workflow_name, asked_when): + label = f'Results from {display_workflow_name(workflow_name)}' + if isinstance(asked_when, str) and asked_when: + return f'{label} ยท you asked on {asked_when}' + return label + + +def assemble_delivery_content(label, body, disclosure=''): + parts = [] + if isinstance(label, str) and label: + parts.append(label) + body_text = body.strip() if isinstance(body, str) else '' + if body_text: + parts.append(body_text) + if isinstance(disclosure, str) and disclosure: + parts.append(disclosure) + return '\n\n'.join(parts) + + +_FAILURE_REASONS = { + 'failed': 'the run stopped before it finished', + 'invalid': "the workflow couldn't run as defined", + 'incomplete': "some of its steps didn't finish", + 'skipped': 'no new or changed files were detected', + REASON_DEADLINE_EXCEEDED: 'it reached its time limit', + 'execution_budget_exceeded': 'it reached its execution limit', + 'repeat_iteration_limit': 'it reached its repeat limit', + 'm365_authorization': 'Microsoft 365 access needs to be reconnected', + 'authorization': 'your workflow access must be restored', +} +FAILURE_REASON_CODES = frozenset(_FAILURE_REASONS) + + +def failure_reason_text(code): + return _FAILURE_REASONS.get(code, _FAILURE_REASONS['failed']) + + +def delivery_note_text(kind, workflow_name, *, reason_code=None): + name = display_workflow_name(workflow_name) + if kind == KIND_FAILED: + return f'{name} failed: {failure_reason_text(reason_code)}.' + if kind == KIND_CANCELLED: + return f'{name} was cancelled.' + if kind == KIND_STATUS: + return f'{name} finished. Open the run to see its results.' + if kind == KIND_CONTENT_BLOCKED: + return f"{name} finished, but its results can't be shown here. Open the run to see them." + if kind == KIND_ANALYSIS: + return f'{name} finished with a saved analysis. Ask a follow-up question about it here.' + if kind == KIND_SKIPPED: + return f"{name} didn't run: no new or changed files were detected." + raise ValueError('unsupported delivery note kind') + + +def notice_workflow_name(name): + text = clean_catalog_text(name, NAME_MAX_LENGTH).replace('"', "'") + if not text: + return 'your workflow' + return f'"{text}"' + + +def _sentence_start(text): + if text.startswith('your workflow'): + return 'Your workflow' + text[len('your workflow'):] + return text + + +def notice_title(kind, workflow_name): + q = notice_workflow_name(workflow_name) + if kind in {KIND_RESULT, KIND_ANALYSIS, KIND_STATUS, KIND_CONTENT_BLOCKED}: + return _sentence_start(f'Results from {q} are ready') + if kind == KIND_FAILED: + return _sentence_start(f"{q} didn't finish") + if kind == KIND_CANCELLED: + return _sentence_start(f'{q} was cancelled') + if kind == KIND_SKIPPED: + return _sentence_start(f"{q} didn't run") + if kind == KIND_EXPIRED: + return _sentence_start(f"{q} didn't finish in time to post to chat") + raise ValueError('unsupported notice kind') + + +def delivered_notice_preview(workflow_name): + return _sentence_start(f'Results from {notice_workflow_name(workflow_name)} are in your chat') + + +def delivery_notification_key(run_id, generation): + return f'workflow-chat-delivery:{run_id}:{generation}' + + +def undeliverable_notification_key(run_id, generation): + return f'workflow-chat-delivery-notice:{run_id}:{generation}' + + +def expired_notification_key(run_id): + return f'workflow-chat-delivery-notice:{run_id}:expired' + + +def token_usage_idempotency_key(message_id, attempt): + return f'workflow_result_delivery:{message_id}:{attempt}' + + +def workflow_run_notice_link(workflow_id, run_id): + return '/workflow-activity?' + urlencode({'workflowId': workflow_id, 'runId': run_id, 'scope': WORKFLOW_SCOPE}) + + +def notice_metadata(workflow_id, run_id, delivery_status): + return {'workflow_id': workflow_id, 'run_id': run_id, 'workflow_scope': WORKFLOW_SCOPE, 'delivery_status': delivery_status} + + +def build_delivery_metadata(*, kind, workflow_id, run_id, generation, run_status, orchestration_run_id, step_id, requested_at): + return { + 'version': CHAT_DELIVERY_VERSION, + 'kind': kind, + 'workflow_id': workflow_id, + 'workflow_scope': WORKFLOW_SCOPE, + 'run_id': run_id, + 'generation': generation, + 'run_status': run_status, + 'orchestration_run_id': orchestration_run_id, + 'step_id': step_id, + 'requested_at': requested_at, + } + + +def next_backoff_seconds(attempts): + if type(attempts) is not int: + return BACKOFF_SECONDS[0] + index = min(max(attempts, 1), len(BACKOFF_SECONDS)) - 1 + return BACKOFF_SECONDS[index] + + +def signal_workflow_chat_delivery(user_id, run_id): + try: + if not isinstance(user_id, str) or not user_id.strip() or not isinstance(run_id, str) or not run_id.strip(): + return False + key = (user_id, run_id) + with _HINT_LOCK: + if key in _HINT_KEYS: + _HINT_EVENT.set() + return True + if len(_HINT_QUEUE) >= HINT_QUEUE_MAX: + return False + _HINT_QUEUE.append(key) + _HINT_KEYS.add(key) + _HINT_EVENT.set() + return True + except Exception: + return False + + +def drain_workflow_chat_delivery_hints(limit=None): + with _HINT_LOCK: + if type(limit) is not int or limit < 0: + limit = len(_HINT_QUEUE) + drained = [] + while _HINT_QUEUE and len(drained) < limit: + key = _HINT_QUEUE.popleft() + _HINT_KEYS.discard(key) + drained.append(key) + if not _HINT_QUEUE: + _HINT_EVENT.clear() + return drained + + +def wait_for_workflow_chat_delivery_hint(timeout): + return _HINT_EVENT.wait(timeout) + + +def clear_workflow_chat_delivery_hints(): + with _HINT_LOCK: + _HINT_QUEUE.clear() + _HINT_KEYS.clear() + _HINT_EVENT.clear() + + +def delivery_log(message, *, level=logging.INFO, status_route=False, run_id=None, conversation_id=None, **fields): + try: + prefix = STATUS_LOG_PREFIX if status_route else LOG_PREFIX + stage = 'workflow_chat_delivery_status' if status_route else 'workflow_chat_delivery' + extra = {'stage': stage, **workflow_log_context(conversation_id=conversation_id, run_id=run_id)} + extra.update({key: value for key, value in fields.items() if value is not None}) + log_event(f'{prefix} {message}', extra=extra, level=level) + except Exception: + return + + +__all__ = ( + 'CHAT_DELIVERY_VERSION', + 'CHAT_DELIVERY_KEY', + 'DELIVERY_METADATA_KEY', + 'DELIVERY_MESSAGE_ID_PREFIX', + 'NOTIFICATION_TYPE', + 'WORKFLOW_SCOPE', + 'CHAT_TRIGGER_SOURCE', + 'LOG_PREFIX', + 'STATUS_LOG_PREFIX', + 'STATUS_PENDING', + 'STATUS_READY', + 'STATUS_DELIVERING', + 'STATUS_DELIVERED', + 'STATUS_UNDELIVERABLE', + 'STATUS_EXPIRED', + 'STATUSES', + 'OPEN_STATUSES', + 'FINAL_STATUSES', + 'REOPENABLE_STATUSES', + 'KIND_RESULT', + 'KIND_FAILED', + 'KIND_CANCELLED', + 'KIND_STATUS', + 'KIND_CONTENT_BLOCKED', + 'KIND_ANALYSIS', + 'KIND_EXPIRED', + 'KIND_SKIPPED', + 'RESULT_CONTEXT_KINDS', + 'PHASE_CLAIMED', + 'PHASE_PUBLISHING', + 'PHASE_UNREAD_MARKED', + 'PHASE_MESSAGE_CREATED', + 'PHASE_NOTIFIED', + 'PHASES', + 'NOTICE_CHAT_RESPONSE', + 'NOTICE_UNDELIVERABLE', + 'NOTICE_EXPIRED', + 'NOTICE_NONE', + 'NOTICE_KINDS', + 'REASON_CHAT_UNAVAILABLE', + 'REASON_ACCESS_LOST', + 'REASON_WORKFLOW_DELETED', + 'REASON_RUNTIME_MISSING', + 'REASON_DELIVERY_FAILED', + 'REASON_EXPIRED_BEFORE_DELIVERY', + 'REASON_CONTENT_BLOCKED', + 'REASON_RESULTS_OFF', + 'REASON_RESULT_UNAVAILABLE', + 'REASON_DEADLINE_EXCEEDED', + 'SILENT_REASONS', + 'RUNTIME_TERMINAL_STATES', + 'RESULT_RUN_STATES', + 'FAILED_RUN_STATES', + 'RUNTIME_RESUMABLE_STATES', + 'RUNTIME_WAITING_STATES', + 'RUNTIME_ACTIVE_STATES', + 'RUN_DOCUMENT_TERMINAL_STATUSES', + 'KIND_BY_TERMINAL_STATE', + 'MODEL_IDENTITY_FIELDS', + 'DELIVERY_GRACE_SECONDS', + 'DEFAULT_DEADLINE_SECONDS', + 'MAX_DEADLINE_SECONDS', + 'LEASE_SECONDS', + 'SWEEP_INTERVAL_SECONDS', + 'SWEEP_LOCK_NAME', + 'SWEEP_LOCK_SECONDS', + 'SWEEP_TOP', + 'SWEEP_MAX_RUNS', + 'SWEEP_MAX_SECONDS', + 'SWEEP_TS_GRACE_SECONDS', + 'BACKOFF_SECONDS', + 'MAX_ATTEMPTS', + 'DEFER_RETRY_SECONDS', + 'DEFER_MAX_SECONDS', + 'STREAM_META_FRESH_SECONDS', + 'RECENT_USER_MESSAGE_SECONDS', + 'HINT_QUEUE_MAX', + 'HISTORY_MAX', + 'MAX_ROLES', + 'MAX_ROLE_LENGTH', + 'MAX_GROUPS', + 'MAX_GROUP_ID_LENGTH', + 'MAX_MODEL_FIELD_LENGTH', + 'MAX_REASONING_EFFORT_LENGTH', + 'NAME_MAX_LENGTH', + 'COMPOSE_TEMPERATURE', + 'COMPOSE_MAX_TOKENS', + 'REPLY_MAX_CHARS', + 'TRIMMED_FALLBACK_CHARS', + 'EXCERPT_BUDGET_BYTES', + 'TRIMMED_FALLBACK_INTRO', + 'FALLBACK_QUESTION', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY', + 'UNDELIVERABLE_NOTICE_MESSAGE', + 'EXPIRED_NOTICE_MESSAGE', + 'DELIVERY_RETRY_UNSUPPORTED', + 'DELIVERY_EDIT_UNSUPPORTED', + 'clean_catalog_text', + 'quoted_workflow_name', + 'display_workflow_name', + 'utc_now', + 'format_delivery_timestamp', + 'parse_delivery_timestamp', + 'workflow_delivery_message_id', + 'delivery_thread_id', + 'is_workflow_delivery_message', + 'workflow_delivery_refusal_payload', + 'normalize_requester_roles', + 'normalize_model_selection', + 'normalize_time_zone', + 'build_chat_delivery_seed', + 'control_summary', + 'compute_expires_at', + 'finalize_chat_delivery_seed', + 'chat_delivery_applies', + 'needs_guarded_run_save', + 'merge_stored_run_fields', + 'reconcile_chat_delivery', + 'phase_at_least', + 'delivery_label', + 'assemble_delivery_content', + 'delivery_note_text', + 'FAILURE_REASON_CODES', + 'failure_reason_text', + 'notice_workflow_name', + 'notice_title', + 'delivered_notice_preview', + 'delivery_notification_key', + 'undeliverable_notification_key', + 'expired_notification_key', + 'token_usage_idempotency_key', + 'workflow_run_notice_link', + 'notice_metadata', + 'build_delivery_metadata', + 'next_backoff_seconds', + 'signal_workflow_chat_delivery', + 'drain_workflow_chat_delivery_hints', + 'wait_for_workflow_chat_delivery_hint', + 'clear_workflow_chat_delivery_hints', + 'delivery_log', +) diff --git a/application/single_app/functions_workflow_chat_delivery_status.py b/application/single_app/functions_workflow_chat_delivery_status.py new file mode 100644 index 000000000..e3341989d --- /dev/null +++ b/application/single_app/functions_workflow_chat_delivery_status.py @@ -0,0 +1,698 @@ +# functions_workflow_chat_delivery_status.py +"""Owner-only status rows for the workflow runs a user's chats started. + +Version: 0.261.227 +Implemented in: 0.261.227 + +Backs ``GET /api/v2/orchestration/workflow-runs/status``, the batched route the V2 run card polls. +Every read is a single-partition query in the signed-in user's own partition that projects only +the fields a row needs. In-flight rows get a bounded number of live runtime reads; finished rows +use the runtime projection stored on the run. Nothing is written, and nothing in a row comes from +a run's output, its error text, or an exception. +""" + +import logging +import re +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Callable + +from functions_m365_workflow_binding import M365_WAITING_STATES +from functions_workflow_alert_safety import sanitize_workflow_alert_record +from functions_workflow_chat_delivery import ( + CHAT_TRIGGER_SOURCE, + DELIVERY_MESSAGE_ID_PREFIX, + FAILED_RUN_STATES, + FAILURE_REASON_CODES, + NAME_MAX_LENGTH, + OPEN_STATUSES, + REASON_ACCESS_LOST, + REASON_CHAT_UNAVAILABLE, + REASON_CONTENT_BLOCKED, + REASON_DEADLINE_EXCEEDED, + REASON_DELIVERY_FAILED, + REASON_EXPIRED_BEFORE_DELIVERY, + REASON_RESULT_UNAVAILABLE, + REASON_RESULTS_OFF, + REASON_RUNTIME_MISSING, + REASON_WORKFLOW_DELETED, + RESULT_RUN_STATES, + RUN_DOCUMENT_TERMINAL_STATUSES, + RUNTIME_ACTIVE_STATES, + RUNTIME_RESUMABLE_STATES, + RUNTIME_TERMINAL_STATES, + RUNTIME_WAITING_STATES, + STATUS_DELIVERED, + STATUS_EXPIRED, + STATUS_READY, + STATUSES, + WORKFLOW_SCOPE, + clean_catalog_text, + delivery_log, + failure_reason_text, + parse_delivery_timestamp, + utc_now, +) +from functions_workflow_definitions import workflow_definition_for_editor, workflow_definition_revision + + +CONVERSATION_LIMIT = 20 +GLOBAL_LIMIT = 50 +LIVE_READ_LIMIT = 10 +WORKFLOW_READ_LIMIT = 20 +RECENT_DELIVERY_SECONDS = 600 + +INVALID_CONVERSATION_CODE = 'invalid_conversation_id' +STATUS_UNAVAILABLE_CODE = 'workflow_run_status_unavailable' +_ERRORS = { + INVALID_CONVERSATION_CODE: ('The conversation ID is not valid.', 400), + STATUS_UNAVAILABLE_CODE: ("Workflow run status isn't available right now. Try again later.", 503), +} + +ROW_STATUSES = ('queued', 'running', 'waiting', 'completed', 'completed_partial', 'failed', 'cancelled', 'expired') +ROW_PHASES = ('running', 'needs_you', 'finished', 'failed', 'cancelled') +DELIVERY_NOT_APPLICABLE = 'not_applicable' +DELIVERY_ROW_STATUSES = ('pending', 'delivering', 'delivered', 'undeliverable', 'expired', DELIVERY_NOT_APPLICABLE) +DELIVERY_REASONS = frozenset({ + REASON_CHAT_UNAVAILABLE, + REASON_ACCESS_LOST, + REASON_WORKFLOW_DELETED, + REASON_RUNTIME_MISSING, + REASON_DELIVERY_FAILED, + REASON_EXPIRED_BEFORE_DELIVERY, + REASON_CONTENT_BLOCKED, + REASON_RESULTS_OFF, + REASON_RESULT_UNAVAILABLE, + REASON_DEADLINE_EXCEEDED, +}) + +RETRY_BLOCKED_WORKFLOW_DELETED = 'workflow_deleted' +RETRY_BLOCKED_ALREADY_RUNNING = 'workflow_already_running' +RETRY_BLOCKED_DEFINITION_CHANGED = 'workflow_definition_changed' +RETRY_BLOCKED_NOT_RESUMABLE = 'not_resumable' +RETRY_BLOCKED_DEADLINE_EXCEEDED = 'deadline_exceeded' +RETRY_BLOCKED_UNAVAILABLE = 'retry_unavailable' +RETRY_BLOCKED_CODES = frozenset({ + RETRY_BLOCKED_WORKFLOW_DELETED, + RETRY_BLOCKED_ALREADY_RUNNING, + RETRY_BLOCKED_DEFINITION_CHANGED, + RETRY_BLOCKED_NOT_RESUMABLE, + RETRY_BLOCKED_DEADLINE_EXCEEDED, + RETRY_BLOCKED_UNAVAILABLE, +}) + +WAITING_ACTION_APPROVE = 'approve' +WAITING_ACTION_RECONNECT = 'reconnect' +WAITING_ACTION_OPEN_RUN = 'open_run' + +_KNOWN_STATES = RUNTIME_ACTIVE_STATES | RUNTIME_WAITING_STATES | RUNTIME_TERMINAL_STATES | frozenset({'canceled'}) +_CANCELLED_STATES = frozenset({'cancelled', 'canceled'}) +_DEFAULT_WORKFLOW_NAME = 'Workflow' +_CONVERSATION_ID = re.compile(r'[A-Za-z0-9_-]{1,128}') +_CODE_MAX_LENGTH = 64 +_ID_MAX_LENGTH = 256 +_SECONDS_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + +_READ_OK = 'ok' +_READ_GONE = 'gone' +_READ_DELETING = 'deleting' +_READ_ERROR = 'error' +_READ_OVER_CAP = 'over_cap' + +_PROJECTION = ', '.join(( + 'c.id AS run_id', + 'c.workflow_id AS workflow_id', + 'c.workflow_name AS workflow_name', + 'c.status AS status', + 'c.started_at AS started_at', + 'c.completed_at AS completed_at', + 'c.progress AS progress', + 'c.runtime_version AS runtime_version', + 'c.definition_revision AS definition_revision', + 'c.chat_invocation.conversation_id AS conversation_id', + 'c.chat_invocation.orchestration_run_id AS orchestration_run_id', + 'c.chat_invocation.step_id AS step_id', + 'c.chat_invocation.requested_at AS requested_at', + 'c.runtime.version AS projection_version', + 'c.runtime.state AS projection_state', + 'c.runtime.phase AS projection_phase', + 'c.runtime.progress AS projection_progress', + 'c.runtime.can_resume AS projection_can_resume', + 'c.runtime.deleted AS projection_deleted', + 'c.runtime.limits.deadline_at AS projection_deadline_at', + 'c.runtime.gate.id AS projection_gate_id', + 'c.runtime.gate.reason_code AS projection_gate_reason_code', + 'c.chat_delivery.status AS delivery_status', + 'c.chat_delivery.generation AS delivery_generation', + 'c.chat_delivery.message_id AS delivery_message_id', + 'c.chat_delivery.delivered_at AS delivery_delivered_at', + 'c.chat_delivery.outcome_reason AS delivery_outcome_reason', +)) +_OWNER_FILTER = 'c.user_id = @user_id AND c.trigger_source = @source AND IS_DEFINED(c.chat_invocation)' +_NEWEST_FIRST = 'ORDER BY c.chat_invocation.requested_at DESC' +CONVERSATION_QUERY = ( + f'SELECT TOP {CONVERSATION_LIMIT + 1} {_PROJECTION} FROM c ' + f'WHERE {_OWNER_FILTER} AND c.chat_invocation.conversation_id = @conversation_id ' + f'{_NEWEST_FIRST}' +) +GLOBAL_QUERY = ( + f'SELECT TOP {GLOBAL_LIMIT + 1} {_PROJECTION} FROM c ' + f'WHERE {_OWNER_FILTER} ' + 'AND (NOT ARRAY_CONTAINS(@terminal, c.status) ' + 'OR ARRAY_CONTAINS(@open, c.chat_delivery.status) ' + 'OR (IS_STRING(c.chat_delivery.delivered_at) AND c.chat_delivery.delivered_at >= @recent)) ' + f'{_NEWEST_FIRST}' +) + + +class WorkflowRunStatusError(Exception): + """A closed status-route failure: a code, its HTTP status and a fixed sentence.""" + + def __init__(self, code): + if code not in _ERRORS: + code = STATUS_UNAVAILABLE_CODE + super().__init__(code) + self.code = code + self.status = _ERRORS[code][1] + + def payload(self): + return {'error': _ERRORS[self.code][0], 'code': self.code}, self.status + + +@dataclass(frozen=True) +class WorkflowRunStatusServices: + """The reads one status request needs. Tests bind fakes; the route binds the real ones.""" + + runs: Any + workflows: Any + get_settings: Callable[[], Any] + settings_gate: Callable[[Any], Any] + live_status: Callable[..., Any] + clock: Callable[[], datetime] = utc_now + + +def default_workflow_run_status_services(): + """Bind the personal workflow containers and runtime reader on first use. + + Imported here so importing this module never loads config or the runtime. + """ + import config + from functions_orchestration_workflow_context import workflow_run_settings_gate + from functions_settings import get_settings + from functions_workflow_runtime import workflow_runtime_status + + return WorkflowRunStatusServices( + runs=config.cosmos_personal_workflow_runs_container, + workflows=config.cosmos_personal_workflows_container, + get_settings=get_settings, + settings_gate=workflow_run_settings_gate, + live_status=workflow_runtime_status, + clock=utc_now, + ) + + +def _identifier(value, max_length=_ID_MAX_LENGTH): + """A stored id as it is, or None: the same shape the runtime store accepts for its ids.""" + if ( + not isinstance(value, str) + or not 0 < len(value) <= max_length + or value != value.strip() + or any(ord(char) < 32 or 0xD800 <= ord(char) <= 0xDFFF for char in value) + ): + return None + return value + + +def _code(value): + if not isinstance(value, str): + return None + text = value.strip().lower() + if not text or len(text) > _CODE_MAX_LENGTH: + return None + return text + + +def _known_state(value): + state = _code(value) + return state if state in _KNOWN_STATES else None + + +def _count(value): + return value if type(value) is int and value >= 0 else None + + +def _seconds(value): + moment = parse_delivery_timestamp(value) + return moment.strftime(_SECONDS_FORMAT) if moment is not None else None + + +def parse_status_conversation_id(values): + """The one optional ``conversation_id`` query value, trimmed; a repeated or malformed one is refused.""" + if values is None: + return None + if isinstance(values, str): + values = [values] + values = list(values) + if not values: + return None + if len(values) > 1: + raise WorkflowRunStatusError(INVALID_CONVERSATION_CODE) + text = values[0].strip() if isinstance(values[0], str) else '' + if not _CONVERSATION_ID.fullmatch(text): + raise WorkflowRunStatusError(INVALID_CONVERSATION_CODE) + return text + + +def status_query(user_id, conversation_id, now): + """The query text, its parameters and the row limit. One partition, newest request first.""" + parameters = [ + {'name': '@user_id', 'value': user_id}, + {'name': '@source', 'value': CHAT_TRIGGER_SOURCE}, + ] + if conversation_id is not None: + parameters.append({'name': '@conversation_id', 'value': conversation_id}) + return CONVERSATION_QUERY, parameters, CONVERSATION_LIMIT + recent = (now - timedelta(seconds=RECENT_DELIVERY_SECONDS)).strftime('%Y-%m-%dT%H:%M:%S.%fZ') + parameters.extend(( + {'name': '@terminal', 'value': sorted(RUN_DOCUMENT_TERMINAL_STATUSES)}, + {'name': '@open', 'value': sorted(OPEN_STATUSES)}, + {'name': '@recent', 'value': recent}, + )) + return GLOBAL_QUERY, parameters, GLOBAL_LIMIT + + +class _WorkflowRead: + __slots__ = ('kind', 'workflow') + + def __init__(self, kind, workflow=None): + self.kind = kind + self.workflow = workflow + + +class _RequestReads: + """One request's read budget. + + One workflow read per distinct workflow (at most ``WORKFLOW_READ_LIMIT``), counted apart from + the live runtime reads (at most ``LIVE_READ_LIMIT``). + """ + + def __init__(self, services, user_id): + self.services = services + self.user_id = user_id + self.workflows = {} + self.live_reads = 0 + + def workflow(self, workflow_id): + if workflow_id in self.workflows: + return self.workflows[workflow_id] + if len(self.workflows) >= WORKFLOW_READ_LIMIT: + return _WorkflowRead(_READ_OVER_CAP) + read = self._read_workflow(workflow_id) + self.workflows[workflow_id] = read + return read + + def _read_workflow(self, workflow_id): + """The workflow in the form the runtime compares revisions on, or why it can't be used.""" + try: + raw = self.services.workflows.read_item(item=workflow_id, partition_key=self.user_id) + except Exception as exc: + if getattr(exc, 'status_code', None) == 404: + return _WorkflowRead(_READ_GONE) + delivery_log( + 'Workflow read failed.', level=logging.WARNING, status_route=True, + error_type=type(exc).__name__, + ) + return _WorkflowRead(_READ_ERROR) + if not isinstance(raw, Mapping) or raw.get('user_id') != self.user_id or raw.get('group_id'): + return _WorkflowRead(_READ_GONE) + if raw.get('deleting') or raw.get('status') == 'deleting': + return _WorkflowRead(_READ_DELETING) + try: + # The same editor form get_personal_workflow returns, which decide_workflow_runtime compares. + cleaned = {key: value for key, value in raw.items() if not str(key).startswith('_')} + workflow = workflow_definition_for_editor(sanitize_workflow_alert_record(cleaned)) + except Exception as exc: + delivery_log( + 'Workflow read failed.', level=logging.WARNING, status_route=True, + error_type=type(exc).__name__, + ) + return _WorkflowRead(_READ_ERROR) + return _WorkflowRead(_READ_OK, workflow) + + def live(self, workflow, run_id): + if self.live_reads >= LIVE_READ_LIMIT: + return None + self.live_reads += 1 + try: + projection = self.services.live_status(workflow, run_id, reader_user_id=self.user_id) + except Exception as exc: + delivery_log( + 'Live run status read failed; using the stored status.', level=logging.WARNING, + status_route=True, run_id=run_id, error_type=type(exc).__name__, + ) + return None + return projection if isinstance(projection, Mapping) else None + + +def _view(*, version, state, phase, progress, can_resume, deleted, deadline_at, gate_id, gate_reason_code): + return { + 'version': _count(version), + 'state': _known_state(state), + 'phase': _code(phase), + 'progress': progress if isinstance(progress, Mapping) else None, + 'can_resume': can_resume if type(can_resume) is bool else None, + 'deleted': deleted is True, + 'deadline_at': deadline_at, + 'gate_id': _identifier(gate_id), + 'gate_reason_code': _code(gate_reason_code), + } + + +def _stored_view(item): + return _view( + version=item.get('projection_version'), + state=item.get('projection_state'), + phase=item.get('projection_phase'), + progress=item.get('projection_progress'), + can_resume=item.get('projection_can_resume'), + deleted=item.get('projection_deleted'), + deadline_at=item.get('projection_deadline_at'), + gate_id=item.get('projection_gate_id'), + gate_reason_code=item.get('projection_gate_reason_code'), + ) + + +def _live_view(projection): + limits = projection.get('limits') if isinstance(projection.get('limits'), Mapping) else {} + gate = projection.get('gate') if isinstance(projection.get('gate'), Mapping) else {} + return _view( + version=projection.get('version'), + state=projection.get('state'), + phase=projection.get('phase'), + progress=projection.get('progress'), + can_resume=projection.get('can_resume'), + deleted=projection.get('deleted'), + deadline_at=limits.get('deadline_at'), + gate_id=gate.get('id'), + gate_reason_code=gate.get('reason_code'), + ) + + +def _deadline_paused(state, view): + return state == 'paused' and REASON_DEADLINE_EXCEEDED in (view['gate_reason_code'], view['phase']) + + +def _status_and_phase(state, *, expired): + if expired: + return 'expired', 'failed' + if state in ('queued', 'running'): + return state, 'running' + if state in RESULT_RUN_STATES: + return state, 'finished' + if state in FAILED_RUN_STATES or state == 'skipped': + return 'failed', 'failed' + if state in _CANCELLED_STATES: + return 'cancelled', 'cancelled' + if state in RUNTIME_WAITING_STATES and state != 'waiting_recovery': + return 'waiting', 'needs_you' + # cancelling, waiting_recovery, and any state this module doesn't know yet. + return 'running', 'running' + + +def _waiting(state, view): + if state == 'waiting_approval': + reason, action = 'approval', WAITING_ACTION_APPROVE + elif state == 'awaiting_sign_in': + reason, action = 'microsoft_365_reconnect', WAITING_ACTION_RECONNECT + elif state in M365_WAITING_STATES: + reason, action = 'microsoft_365_approval', WAITING_ACTION_OPEN_RUN + elif state == 'waiting_output': + reason, action = 'output_review', WAITING_ACTION_OPEN_RUN + elif state == 'waiting_recovery': + reason, action = 'recovery', WAITING_ACTION_OPEN_RUN + elif state == 'paused': + reason = REASON_DEADLINE_EXCEEDED if _deadline_paused(state, view) else 'paused' + action = WAITING_ACTION_OPEN_RUN + else: + return None + return {'reason': reason, 'action': action, 'gate_id': view['gate_id']} + + +def _steps(*candidates): + """Steps finished and steps in all, from the first ``{completed, total}`` progress that has both.""" + for progress in candidates: + if not isinstance(progress, Mapping): + continue + completed = _count(progress.get('completed')) + total = _count(progress.get('total')) + if completed is not None and total is not None: + return min(completed, total), total + return None, None + + +def _elapsed_seconds(started_at, end): + started = parse_delivery_timestamp(started_at) + if started is None or end is None: + return None + return max(0, int((end - started).total_seconds())) + + +def _delivery(item): + status = item.get('delivery_status') + if status not in STATUSES: + return { + 'status': DELIVERY_NOT_APPLICABLE, + 'generation': None, + 'message_id': None, + 'delivered_at': None, + 'reason': None, + } + delivered = status == STATUS_DELIVERED + message_id = _identifier(item.get('delivery_message_id')) + reason = item.get('delivery_outcome_reason') + return { + 'status': 'pending' if status == STATUS_READY else status, + 'generation': _count(item.get('delivery_generation')), + 'message_id': message_id if delivered and message_id and message_id.startswith(DELIVERY_MESSAGE_ID_PREFIX) else None, + 'delivered_at': _seconds(item.get('delivery_delivered_at')) if delivered else None, + 'reason': reason if reason in DELIVERY_REASONS else None, + } + + +def _failure(status, state, view): + """The closed failure code and its fixed sentence; never the run's own error text.""" + if status == 'expired': + code = REASON_DEADLINE_EXCEEDED + elif status == 'failed': + candidate = view['gate_reason_code'] or state + code = candidate if candidate in FAILURE_REASON_CODES else 'failed' + else: + return None, None + text = failure_reason_text(code) + return text[:1].upper() + text[1:] + '.', code + + +def _retry_blocked(view, runtime_version, item, workflow_read, run_id, delivery_status, now): + """Why the durable resume route would refuse this run now, or None when it should accept it. + + The checks mirror ``decide_workflow_runtime`` and ``journal_request('resume')`` on the stored + projection, so the answer is a snapshot: the resume route can still refuse. + """ + if view['can_resume'] is None or runtime_version is None: + return RETRY_BLOCKED_UNAVAILABLE + if view['deleted']: + return RETRY_BLOCKED_WORKFLOW_DELETED + if not view['can_resume']: + return RETRY_BLOCKED_NOT_RESUMABLE + deadline = parse_delivery_timestamp(view['deadline_at']) + if (deadline is not None and deadline <= now) or delivery_status == STATUS_EXPIRED: + return RETRY_BLOCKED_DEADLINE_EXCEEDED + if workflow_read is None or workflow_read.kind in (_READ_ERROR, _READ_OVER_CAP): + return RETRY_BLOCKED_UNAVAILABLE + if workflow_read.kind in (_READ_GONE, _READ_DELETING): + return RETRY_BLOCKED_WORKFLOW_DELETED + revision = item.get('definition_revision') + if not isinstance(revision, str) or not revision: + return RETRY_BLOCKED_UNAVAILABLE + try: + current = workflow_definition_revision(workflow_read.workflow) + except Exception: + return RETRY_BLOCKED_UNAVAILABLE + if current != revision: + return RETRY_BLOCKED_DEFINITION_CHANGED + if workflow_read.workflow.get('active_run_id') not in (None, '', run_id): + return RETRY_BLOCKED_ALREADY_RUNNING + return None + + +def _status_row(item, reads, now): + if not isinstance(item, Mapping): + return None + run_id = _identifier(item.get('run_id')) + workflow_id = _identifier(item.get('workflow_id')) + conversation_id = _identifier(item.get('conversation_id')) + if not run_id or not workflow_id or not conversation_id: + return None + + stored = _stored_view(item) + stored_state = stored['state'] or _known_state(item.get('status')) + in_flight = stored_state not in RUN_DOCUMENT_TERMINAL_STATUSES + workflow_read = None + if in_flight or stored_state in RUNTIME_RESUMABLE_STATES: + workflow_read = reads.workflow(workflow_id) + + view = stored + live = False + if in_flight and workflow_read.kind == _READ_OK: + projection = reads.live(workflow_read.workflow, run_id) + if projection is not None: + view = _live_view(projection) + live = True + state = view['state'] or stored_state + terminal = state in RUN_DOCUMENT_TERMINAL_STATUSES + + if live and view['version'] is not None: + runtime_version = view['version'] + elif _count(item.get('runtime_version')) is not None: + runtime_version = item['runtime_version'] + else: + runtime_version = stored['version'] + + delivery = _delivery(item) + raw_delivery_status = item.get('delivery_status') + expired = not terminal and (raw_delivery_status == STATUS_EXPIRED or _deadline_paused(state, view)) + status, phase = _status_and_phase(state, expired=expired) + waiting = _waiting(state, view) + error, error_code = _failure(status, state, view) + + retry_blocked = None + if state in RUNTIME_RESUMABLE_STATES: + retry_blocked = _retry_blocked(view, runtime_version, item, workflow_read, run_id, raw_delivery_status, now) + step_index, step_count = _steps(view['progress'], stored['progress'], item.get('progress')) + completed_at = _seconds(item.get('completed_at')) if terminal else None + + return { + 'workflow_id': workflow_id, + 'workflow_scope': WORKFLOW_SCOPE, + 'run_id': run_id, + 'workflow_name': clean_catalog_text(item.get('workflow_name'), NAME_MAX_LENGTH) or _DEFAULT_WORKFLOW_NAME, + 'conversation_id': conversation_id, + 'orchestration_run_id': _identifier(item.get('orchestration_run_id')), + 'step_id': _identifier(item.get('step_id')), + 'requested_at': _seconds(item.get('requested_at')), + 'status': status, + 'phase': phase, + 'runtime_version': runtime_version, + 'step_index': step_index, + 'step_count': step_count, + 'step_label': None, + 'started_at': _seconds(item.get('started_at')), + 'completed_at': completed_at, + 'elapsed_seconds': _elapsed_seconds( + item.get('started_at'), + parse_delivery_timestamp(completed_at) if terminal else now, + ), + 'waiting': waiting, + 'delivery': delivery, + 'error': error, + 'error_code': error_code, + 'retry_blocked': retry_blocked, + 'actions': { + 'cancel': not terminal and not expired and state != 'cancelling', + 'retry': state in RUNTIME_RESUMABLE_STATES and retry_blocked is None, + 'approve': bool( + waiting and waiting['action'] == WAITING_ACTION_APPROVE and waiting['gate_id'] and not expired + ), + 'open_run': True, + }, + 'live': live, + } + + +def _request_now(services): + try: + moment = parse_delivery_timestamp(services.clock()) + except Exception: + moment = None + return moment or utc_now() + + +def workflow_run_status_payload(user_id, conversation_id_values=None, *, services=None): + """The status route's 200 body for ``user_id``, or a ``WorkflowRunStatusError``. + + ``conversation_id_values`` is every ``conversation_id`` the request sent. With one, the rows + are that chat's runs; with none, they are the user's runs still in flight, still waiting to be + delivered, or delivered in the last ten minutes. Rows come back even when chats can no longer + start workflows, so an in-flight card doesn't vanish when an admin changes the setting. + """ + conversation_id = parse_status_conversation_id(conversation_id_values) + if _identifier(user_id) is None: + raise WorkflowRunStatusError(STATUS_UNAVAILABLE_CODE) + services = services or default_workflow_run_status_services() + now = _request_now(services) + try: + available = services.settings_gate(services.get_settings()) is None + except Exception as exc: + delivery_log( + 'Workflow run status settings read failed.', level=logging.WARNING, status_route=True, + error_type=type(exc).__name__, + ) + raise WorkflowRunStatusError(STATUS_UNAVAILABLE_CODE) from None + + query, parameters, limit = status_query(user_id, conversation_id, now) + try: + items = list(services.runs.query_items(query=query, parameters=parameters, partition_key=user_id)) + except Exception as exc: + delivery_log( + 'Workflow run status query failed.', level=logging.WARNING, status_route=True, + conversation_id=conversation_id, error_type=type(exc).__name__, + ) + raise WorkflowRunStatusError(STATUS_UNAVAILABLE_CODE) from None + + reads = _RequestReads(services, user_id) + rows = [] + for item in items[:limit]: + row = _status_row(item, reads, now) + if row is not None: + rows.append(row) + return { + 'available': available, + 'runs': rows, + 'checked_at': now.strftime(_SECONDS_FORMAT), + 'truncated': len(items) > limit, + } + + +__all__ = ( + 'CONVERSATION_LIMIT', + 'GLOBAL_LIMIT', + 'LIVE_READ_LIMIT', + 'WORKFLOW_READ_LIMIT', + 'RECENT_DELIVERY_SECONDS', + 'INVALID_CONVERSATION_CODE', + 'STATUS_UNAVAILABLE_CODE', + 'ROW_STATUSES', + 'ROW_PHASES', + 'DELIVERY_NOT_APPLICABLE', + 'DELIVERY_ROW_STATUSES', + 'DELIVERY_REASONS', + 'RETRY_BLOCKED_WORKFLOW_DELETED', + 'RETRY_BLOCKED_ALREADY_RUNNING', + 'RETRY_BLOCKED_DEFINITION_CHANGED', + 'RETRY_BLOCKED_NOT_RESUMABLE', + 'RETRY_BLOCKED_DEADLINE_EXCEEDED', + 'RETRY_BLOCKED_UNAVAILABLE', + 'RETRY_BLOCKED_CODES', + 'WAITING_ACTION_APPROVE', + 'WAITING_ACTION_RECONNECT', + 'WAITING_ACTION_OPEN_RUN', + 'CONVERSATION_QUERY', + 'GLOBAL_QUERY', + 'WorkflowRunStatusError', + 'WorkflowRunStatusServices', + 'default_workflow_run_status_services', + 'parse_status_conversation_id', + 'status_query', + 'workflow_run_status_payload', +) diff --git a/application/single_app/functions_workflow_chat_delivery_worker.py b/application/single_app/functions_workflow_chat_delivery_worker.py new file mode 100644 index 000000000..771065885 --- /dev/null +++ b/application/single_app/functions_workflow_chat_delivery_worker.py @@ -0,0 +1,1743 @@ +# functions_workflow_chat_delivery_worker.py +"""Deliver a chat-started workflow run's outcome back into the chat that asked (6b-1). + +A chat-orchestration plan that starts one of the user's saved workflows ends right away, and the +run may finish hours later. This worker posts the run's outcome into the requesting chat once per +delivery generation: + +* a reply the model composes from a bounded, fenced excerpt of the run's result, under a dated + label; +* a fixed note when the run failed or was cancelled, or its result can't be shown here; +* nothing, and one bell notice instead, when the chat can't take it any more. + +The record that drives delivery is the run document's ``chat_delivery`` field. Only ETag-guarded +writes that fence on the claim's lease advance it. The message ID, the unread mark and every +notification are idempotent, so a crash at any point resumes without a second post or notice. + +This module imports only the standard library, azure-core's match conditions, the unread outcome +constants and the pure delivery contract, so importing it never initializes application storage. +``default_workflow_chat_delivery_services`` binds the production dependencies on first use. +""" + +import logging +import re +import time +import uuid +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Callable, Optional + +from azure.core import MatchConditions + +from functions_workflow_chat_delivery import ( + CHAT_DELIVERY_KEY, + COMPOSE_MAX_TOKENS, + COMPOSE_TEMPERATURE, + DEFER_MAX_SECONDS, + DEFER_RETRY_SECONDS, + DELIVERY_METADATA_KEY, + EXCERPT_BUDGET_BYTES, + EXPIRED_NOTICE_MESSAGE, + FALLBACK_QUESTION, + KIND_ANALYSIS, + KIND_CANCELLED, + KIND_CONTENT_BLOCKED, + KIND_EXPIRED, + KIND_FAILED, + KIND_RESULT, + KIND_SKIPPED, + KIND_STATUS, + LEASE_SECONDS, + MAX_ATTEMPTS, + NOTICE_CHAT_RESPONSE, + NOTICE_EXPIRED, + NOTICE_NONE, + NOTICE_UNDELIVERABLE, + NOTIFICATION_TYPE, + PHASE_CLAIMED, + PHASE_MESSAGE_CREATED, + PHASE_NOTIFIED, + PHASE_PUBLISHING, + PHASE_UNREAD_MARKED, + REASON_ACCESS_LOST, + REASON_CHAT_UNAVAILABLE, + REASON_CONTENT_BLOCKED, + REASON_DELIVERY_FAILED, + REASON_EXPIRED_BEFORE_DELIVERY, + REASON_RESULT_UNAVAILABLE, + REASON_RESULTS_OFF, + REASON_WORKFLOW_DELETED, + RECENT_USER_MESSAGE_SECONDS, + REPLY_MAX_CHARS, + RESULT_CONTEXT_KINDS, + RUN_DOCUMENT_TERMINAL_STATUSES, + SILENT_REASONS, + STATUS_DELIVERED, + STATUS_DELIVERING, + STATUS_EXPIRED, + STATUS_PENDING, + STATUS_READY, + STATUS_UNDELIVERABLE, + STREAM_META_FRESH_SECONDS, + SWEEP_LOCK_NAME, + SWEEP_LOCK_SECONDS, + SWEEP_MAX_RUNS, + SWEEP_MAX_SECONDS, + SWEEP_TOP, + SWEEP_TS_GRACE_SECONDS, + TRIMMED_FALLBACK_CHARS, + TRIMMED_FALLBACK_INTRO, + UNDELIVERABLE_NOTICE_MESSAGE, + assemble_delivery_content, + build_delivery_metadata, + control_summary, + delivered_notice_preview, + delivery_label, + delivery_log, + delivery_note_text, + delivery_notification_key, + delivery_thread_id, + expired_notification_key, + format_delivery_timestamp, + next_backoff_seconds, + notice_metadata, + notice_title, + parse_delivery_timestamp, + phase_at_least, + reconcile_chat_delivery, + token_usage_idempotency_key, + undeliverable_notification_key, + utc_now, + workflow_delivery_message_id, + workflow_run_notice_link, +) +from functions_workflow_chat_delivery import _close_for_runtime +from functions_conversation_unread import ( + UNREAD_ALREADY_MARKED, + UNREAD_MARKED, + UNREAD_NOT_FOUND, + UNREAD_READ_SINCE, + UNREAD_UNAVAILABLE, +) + + +QUESTION_MAX_CHARS = 4000 +RECORD_WRITE_RETRIES = 5 +HINT_BATCH = 32 +NEWEST_MESSAGE_QUERY = ( + 'SELECT TOP 1 c.id, c.role, c.timestamp, c.metadata.thread_info AS thread_info ' + 'FROM c WHERE c.conversation_id = @conversation_id ORDER BY c.timestamp DESC' +) + +OUTCOME_DELIVERED = 'delivered' +OUTCOME_UNDELIVERABLE = 'undeliverable' +OUTCOME_CLOSED_SILENTLY = 'closed_silently' +OUTCOME_EXPIRED = 'expired' +OUTCOME_DEFERRED = 'deferred' +OUTCOME_RETRY_SCHEDULED = 'retry_scheduled' +OUTCOME_PENDING = 'pending' +OUTCOME_NOT_DUE = 'not_due' +OUTCOME_BUSY = 'busy' +OUTCOME_LOST_LEASE = 'lost_lease' +OUTCOME_NOT_APPLICABLE = 'not_applicable' +OUTCOME_UNAVAILABLE = 'unavailable' + + +class _Transient(Exception): + """A retryable failure. It consumes the claim's attempt and backs off.""" + + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +class _LostLease(Exception): + """Another worker owns the record now, or the run document is gone. Stop without writing.""" + + +class _Defer(Exception): + """The chat may be mid-reply. ``counted`` is True when the deferral consumes the attempt.""" + + def __init__(self, counted): + super().__init__('deferred') + self.counted = counted + + +@dataclass +class WorkflowChatDeliveryServices: + """Every dependency the worker touches, so tests can drive it without Azure.""" + + runs: Any + workflows: Any + conversations: Any + messages: Any + get_settings: Callable + runtime_store_factory: Callable + runtime_conflict_error: type + runtime_unavailable_error: type + read_workflow_result: Callable + result_unavailable_error: type + workflow_result_context: Callable + format_run_time: Callable + format_disclosure: Callable + build_result_messages: Callable + new_fence_nonce: Callable + excerpt_budget_steps: tuple + calculate_budget: Callable + resolve_model: Callable + model_error: type + capture_identity: Callable + extract_text: Callable + check_content: Callable + attach_chat_check: Callable + record_incident: Callable + is_message_retracted: Callable + create_chat_response_notification: Callable + create_notification: Callable + log_token_usage: Callable + bump_cache_version: Callable + mark_unread_guarded: Callable + is_conversation_private: Callable + is_user_workflows_enabled: Callable + is_chat_workflow_results_enabled: Callable + stream_activity_probe: Callable + acquire_lock: Callable + release_lock: Callable + drain_hints: Callable + signal_delivery: Callable + clock: Callable + compose_output_tokens: Callable = None + monotonic: Callable = time.monotonic + + +def _no_lock(_task_name, _lease_seconds): + return None + + +def _ignore_release(_lock_document): + return None + + +def _personal_runtime_store(user_id, workflow_id, run_id): + # A personal store's identity is the workflow ID and owner only, so a tombstoned control stays + # readable after the workflow document itself is gone. + from functions_workflow_runtime_store import workflow_runtime_store + + return workflow_runtime_store({'id': workflow_id, 'user_id': user_id}, run_id) + + +def _default_stream_activity_probe(user_id, conversation_id, now): + """Whether a reply may be streaming into this chat right now (D7 signals 1 and 2).""" + import sys + + chats = sys.modules.get('route_backend_chats') + registry = getattr(chats, 'CHAT_STREAM_REGISTRY', None) if chats is not None else None + if registry is not None: + try: + if registry.get_session(user_id, conversation_id, active_only=True): + return True + except Exception as exc: + delivery_log('Stream registry probe failed.', level=logging.WARNING, error_type=type(exc).__name__) + try: + import app_settings_cache + + getter = getattr(app_settings_cache, 'get_stream_session_meta', None) + meta = getter(f'{user_id}:{conversation_id}') if callable(getter) else None + except Exception as exc: + delivery_log('Stream metadata probe failed.', level=logging.WARNING, error_type=type(exc).__name__) + meta = None + if isinstance(meta, dict) and meta.get('active'): + updated = parse_delivery_timestamp(meta.get('updated_at')) + if updated is not None and (now - updated).total_seconds() < STREAM_META_FRESH_SECONDS: + return True + return False + + +def default_workflow_chat_delivery_services(*, acquire_lock=None, release_lock=None): + """Bind the production dependencies. + + Every import is deferred to this call because each one initializes application storage. The + sweep's distributed lock helpers come from background_tasks, which passes them in. Without them + the sweep never runs; hint-driven deliveries still do, because a claim is an ETag write. + """ + import config + import functions_orchestration_models as orchestration_models + from agent_execution_context import capture_execution_identity + from functions_activity_logging import log_token_usage + from functions_chat_content_checks import attach_chat_check, check_chat_content + from functions_chat_content_review import record_chat_content_incident, reply_is_retracted + from functions_conversation_cache import bump_conversation_cache_version + from functions_conversation_unread import mark_conversation_unread_guarded + from functions_notifications import create_chat_response_notification, create_notification + from functions_orchestration_memory import conversation_is_private + from functions_settings import ( + get_settings, + is_chat_workflow_results_enabled_for_user, + is_user_workflows_enabled_for_user, + ) + from functions_workflow_chat_delivery import ( + drain_workflow_chat_delivery_hints, + signal_workflow_chat_delivery, + ) + from functions_workflow_context import calculate_workflow_context_budget + from functions_workflow_result_followup import ( + EXCERPT_BUDGET_STEPS, + build_workflow_result_messages, + new_fence_nonce, + ) + from functions_workflow_result_reader import ( + WorkflowResultUnavailable, + format_workflow_result_disclosure, + format_workflow_run_time, + read_workflow_result, + workflow_result_context, + ) + from functions_workflow_runtime_store import RuntimeUnavailable, WorkflowRuntimeConflict + from model_endpoint_clients import ModelEndpointBehavior, extract_chat_completion_response_text + + def compose_output_tokens(model): + # Mirrors OrchestrationModel.create_completion: reasoning tokens share the completion budget. + behavior = ModelEndpointBehavior(model.provider, getattr(model, 'behavior_name', None) or model.deployment) + if behavior.is_openai_reasoning_model: + return max(COMPOSE_MAX_TOKENS, orchestration_models.REASONING_COMPLETION_BUDGET) + return COMPOSE_MAX_TOKENS + + return WorkflowChatDeliveryServices( + runs=config.cosmos_personal_workflow_runs_container, + workflows=config.cosmos_personal_workflows_container, + conversations=config.cosmos_conversations_container, + messages=config.cosmos_messages_container, + get_settings=get_settings, + runtime_store_factory=_personal_runtime_store, + runtime_conflict_error=WorkflowRuntimeConflict, + runtime_unavailable_error=RuntimeUnavailable, + read_workflow_result=read_workflow_result, + result_unavailable_error=WorkflowResultUnavailable, + workflow_result_context=workflow_result_context, + format_run_time=format_workflow_run_time, + format_disclosure=format_workflow_result_disclosure, + build_result_messages=build_workflow_result_messages, + new_fence_nonce=new_fence_nonce, + excerpt_budget_steps=tuple(EXCERPT_BUDGET_STEPS), + calculate_budget=calculate_workflow_context_budget, + resolve_model=orchestration_models.resolve_orchestration_model, + model_error=orchestration_models.OrchestrationModelError, + capture_identity=capture_execution_identity, + extract_text=extract_chat_completion_response_text, + check_content=check_chat_content, + attach_chat_check=attach_chat_check, + record_incident=record_chat_content_incident, + is_message_retracted=reply_is_retracted, + create_chat_response_notification=create_chat_response_notification, + create_notification=create_notification, + log_token_usage=log_token_usage, + bump_cache_version=bump_conversation_cache_version, + mark_unread_guarded=mark_conversation_unread_guarded, + is_conversation_private=conversation_is_private, + is_user_workflows_enabled=is_user_workflows_enabled_for_user, + is_chat_workflow_results_enabled=is_chat_workflow_results_enabled_for_user, + stream_activity_probe=_default_stream_activity_probe, + acquire_lock=acquire_lock or _no_lock, + release_lock=release_lock or _ignore_release, + drain_hints=drain_workflow_chat_delivery_hints, + signal_delivery=signal_workflow_chat_delivery, + clock=utc_now, + compose_output_tokens=compose_output_tokens, + ) + + +def _status_code(exc): + code = getattr(exc, 'status_code', None) + return code if type(code) is int else None + + +def _message_timestamp(moment): + # Naive ISO UTC with microseconds: the format both conversation loaders already sort. + return format_delivery_timestamp(moment)[:-1] + + +def _now(services): + return parse_delivery_timestamp(services.clock()) or utc_now() + + +def _later(moment, seconds): + return format_delivery_timestamp(moment + timedelta(seconds=seconds)) + + +def _count(value): + return value if type(value) is int and value >= 0 else 0 + + +class _Delivery: + """One claimed generation: the run document as last written and the lease that fences it.""" + + def __init__(self, services, user_id, run, lease_id, *, exhausted=False): + self.services = services + self.user_id = user_id + self.run = run + self.lease_id = lease_id + self.exhausted = exhausted + self.workflow = None + self.settings = None + self.conversation = None + + @property + def run_id(self): + return self.run.get('id') + + @property + def workflow_id(self): + return self.run.get('workflow_id') + + @property + def record(self): + value = self.run.get(CHAT_DELIVERY_KEY) + return dict(value) if isinstance(value, dict) else {} + + @property + def invocation(self): + value = self.run.get('chat_invocation') + return value if isinstance(value, dict) else {} + + @property + def conversation_id(self): + return self.invocation.get('conversation_id') + + @property + def generation(self): + return self.record.get('generation') + + def workflow_name(self): + if isinstance(self.workflow, dict) and isinstance(self.workflow.get('name'), str): + return self.workflow.get('name') + return self.run.get('workflow_name') + + def message_id(self): + return workflow_delivery_message_id(self.run_id, self.conversation_id, self.generation) + + def now(self): + return _now(self.services) + + +def _read_run(services, user_id, run_id): + """The run document, or None when it's gone. Other storage errors propagate.""" + try: + run = services.runs.read_item(item=run_id, partition_key=user_id) + except Exception as exc: + if _status_code(exc) == 404: + return None + raise + if not isinstance(run, dict) or run.get('user_id') != user_id: + return None + return run + + +def _read_control_summary(services, user_id, run): + """Summarize the run's runtime control. A control that was never written reads as missing.""" + try: + store = services.runtime_store_factory(user_id, run.get('workflow_id'), run.get('id')) + control = store.read(allow_deleted=True) + except services.runtime_conflict_error as exc: + if getattr(exc, 'code', None) == 'not_found': + return control_summary(None) + raise _Transient('runtime_conflict') from None + except services.runtime_unavailable_error: + raise _Transient('runtime_unavailable') from None + except Exception as exc: + delivery_log('Runtime control read failed.', level=logging.WARNING, run_id=run.get('id'), error_type=type(exc).__name__) + raise _Transient('runtime_unavailable') from None + return control_summary(control) + + +def _replace_run(services, run, record): + """One ETag-guarded replace of the delivery record: the stored run, or None after a 412.""" + body = dict(run) + body[CHAT_DELIVERY_KEY] = record + try: + stored = services.runs.replace_item( + item=run['id'], body=body, etag=run.get('_etag'), match_condition=MatchConditions.IfNotModified, + ) + except Exception as exc: + code = _status_code(exc) + if code == 412: + return None + if code == 404: + raise _LostLease() from None + delivery_log( + 'Delivery record write failed.', level=logging.WARNING, run_id=run.get('id'), + error_type=type(exc).__name__, code=code, + ) + raise _Transient('record_unavailable') from None + return stored if isinstance(stored, dict) else body + + +def _write_claimed(delivery, mutate, *, release=False, summary=None): + """Apply ``mutate(record, now)`` under the lease and the run's ETag; returns the record written. + + Every write renews the lease unless it releases it. A releasing write is reconciled with + ``summary`` (a fresh control summary) so a resume that landed mid-delivery reopens the record. + """ + services = delivery.services + for _attempt in range(RECORD_WRITE_RETRIES + 1): + current = delivery.run.get(CHAT_DELIVERY_KEY) + if ( + not isinstance(current, dict) or current.get('status') != STATUS_DELIVERING + or current.get('lease_id') != delivery.lease_id + ): + raise _LostLease() + now = delivery.now() + record = mutate(dict(current), now) + record['updated_at'] = format_delivery_timestamp(now) + if release: + record.update({'lease_id': None, 'lease_expires_at': None}) + if summary is not None: + record, _changed, _ready = reconcile_chat_delivery(record, summary, now) + else: + record['lease_expires_at'] = _later(now, LEASE_SECONDS) + stored = _replace_run(services, delivery.run, record) + if stored is not None: + delivery.run = stored + return record + fresh = _read_run(services, delivery.user_id, delivery.run_id) + if fresh is None: + raise _LostLease() + delivery.run = fresh + raise _Transient('record_conflict') + + +def _settled_outcome(record, changed): + status = record.get('status') + if status == STATUS_PENDING: + return OUTCOME_PENDING + if status == STATUS_READY: + return OUTCOME_NOT_DUE + if changed and status in {STATUS_DELIVERED, STATUS_UNDELIVERABLE}: + return OUTCOME_CLOSED_SILENTLY + return OUTCOME_NOT_APPLICABLE + + +def _postpone_unclaimed(services, user_id, run, reason): + """An open record whose control can't be read waits out a backoff, so it never holds a sweep slot. + + The cap closes it without a notice: no outcome was ever read, so there's nothing to tell. + """ + for _attempt in range(RECORD_WRITE_RETRIES + 1): + record = run.get(CHAT_DELIVERY_KEY) if isinstance(run, dict) else None + if not isinstance(record, dict) or record.get('status') not in {STATUS_PENDING, STATUS_READY, STATUS_DELIVERING}: + return OUTCOME_UNAVAILABLE + now = _now(services) + stamp = format_delivery_timestamp(now) + attempts = _count(record.get('attempts')) + 1 + if attempts >= MAX_ATTEMPTS: + updated = _close_for_runtime(record, REASON_DELIVERY_FAILED, stamp) + updated['attempts'] = attempts + else: + updated = dict(record) + updated.update({ + 'status': STATUS_READY if record.get('status') == STATUS_DELIVERING else record.get('status'), + 'lease_id': None, + 'lease_expires_at': None, + 'attempts': attempts, + 'next_attempt_at': _later(now, next_backoff_seconds(attempts)), + 'updated_at': stamp, + }) + stored = _replace_run(services, run, updated) + if stored is not None: + delivery_log( + 'Delivery postponed: the runtime control could not be read.', level=logging.WARNING, + run_id=run.get('id'), reason=reason, attempt=attempts, + ) + if updated.get('status') in {STATUS_DELIVERED, STATUS_UNDELIVERABLE}: + return OUTCOME_CLOSED_SILENTLY + return OUTCOME_RETRY_SCHEDULED + run = _read_run(services, user_id, run.get('id')) + return OUTCOME_BUSY + + +def _claim(services, user_id, run_id): + """Reconcile the record with the runtime, then claim it when a delivery is due. + + Returns a ``_Delivery`` for a claim, otherwise an outcome string. + """ + run = _read_run(services, user_id, run_id) + for _attempt in range(RECORD_WRITE_RETRIES + 1): + record = run.get(CHAT_DELIVERY_KEY) if isinstance(run, dict) else None + if not isinstance(record, dict): + return OUTCOME_NOT_APPLICABLE + now = _now(services) + released = False + if record.get('status') == STATUS_DELIVERING: + lease_end = parse_delivery_timestamp(record.get('lease_expires_at')) + if lease_end is not None and lease_end > now: + return OUTCOME_BUSY + # An abandoned claim is released, then reconciled like any ready record. + record = {**record, 'status': STATUS_READY, 'lease_id': None, 'lease_expires_at': None} + released = True + try: + summary = _read_control_summary(services, user_id, run) + except _Transient as exc: + waiting_until = parse_delivery_timestamp(record.get('next_attempt_at')) + if waiting_until is not None and waiting_until > now: + # An early hint never spends an attempt; the due sweep reads the control again. + return OUTCOME_NOT_DUE + return _postpone_unclaimed(services, user_id, run, exc.reason) + updated, changed, ready = reconcile_chat_delivery(record, summary, now) + due = parse_delivery_timestamp(updated.get('next_attempt_at')) + if ready and (due is None or due <= now): + attempts = _count(updated.get('attempts')) + lease_id = uuid.uuid4().hex + claimed = dict(updated) + claimed.update({ + 'status': STATUS_DELIVERING, + 'lease_id': lease_id, + 'lease_expires_at': _later(now, LEASE_SECONDS), + 'phase': updated.get('phase') or PHASE_CLAIMED, + 'attempts': attempts + 1, + 'next_attempt_at': None, + 'updated_at': format_delivery_timestamp(now), + }) + stored = _replace_run(services, run, claimed) + if stored is not None: + return _Delivery(services, user_id, stored, lease_id, exhausted=attempts >= MAX_ATTEMPTS) + elif changed or released: + stored = _replace_run(services, run, updated) + if stored is not None: + if updated.get('status') == STATUS_READY and due is not None: + return OUTCOME_NOT_DUE + return _settled_outcome(updated, changed) + else: + return _settled_outcome(updated, False) + run = _read_run(services, user_id, run_id) + return OUTCOME_BUSY + + +def _read_workflow(delivery): + """The owner's workflow document, or None once it's gone or being deleted.""" + services = delivery.services + try: + workflow = services.workflows.read_item(item=delivery.workflow_id, partition_key=delivery.user_id) + except Exception as exc: + if _status_code(exc) == 404: + return None + delivery_log( + 'Workflow read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('workflow_unavailable') from None + if not isinstance(workflow, dict) or workflow.get('user_id') != delivery.user_id: + return None + if workflow.get('deleting') or workflow.get('status') == 'deleting': + return None + return workflow + + +def _conversation_ok(services, conversation, user_id): + """Whether the chat can still take a delivery: the owner's, not deleted, and private.""" + if not isinstance(conversation, dict) or conversation.get('user_id') != user_id: + return False + # The same markers the guarded unread mark checks, so both decide alike. + if conversation.get('orchestration_deleted') or conversation.get('deleted'): + return False + try: + return services.is_conversation_private(conversation, user_id) is True + except Exception as exc: + delivery_log('Chat privacy check failed.', level=logging.WARNING, error_type=type(exc).__name__) + raise _Transient('privacy_unavailable') from None + + +def _read_conversation(delivery): + """The requesting chat when it can still take the delivery, otherwise None.""" + services = delivery.services + conversation_id = delivery.conversation_id + if not isinstance(conversation_id, str) or not conversation_id: + return None + try: + conversation = services.conversations.read_item(item=conversation_id, partition_key=conversation_id) + except Exception as exc: + if _status_code(exc) == 404: + return None + delivery_log( + 'Chat read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('conversation_unavailable') from None + if not _conversation_ok(services, conversation, delivery.user_id): + return None + delivery.conversation = conversation + return conversation + + +def _newest_message(delivery): + """The chat's newest message row, or None for an empty chat. + + A failed read is never treated as an empty chat: it defers, and the deferral counts (D7). + """ + services = delivery.services + try: + rows = list(services.messages.query_items( + query=NEWEST_MESSAGE_QUERY, + parameters=[{'name': '@conversation_id', 'value': delivery.conversation_id}], + partition_key=delivery.conversation_id, + )) + except Exception as exc: + delivery_log( + 'Newest-message read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Defer(True) from None + newest = rows[0] if rows else None + return newest if isinstance(newest, dict) else None + + +def _message_exists(delivery, message_id): + """Whether the delivery message was already created by an earlier attempt.""" + services = delivery.services + try: + message = services.messages.read_item(item=message_id, partition_key=delivery.conversation_id) + except Exception as exc: + if _status_code(exc) == 404: + return False + delivery_log( + 'Delivery message read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('message_unavailable') from None + return isinstance(message, dict) + + +def _defer_probe(delivery, record, now): + """Raise ``_Defer`` while a reply may be streaming into the chat (D7). + + Fifteen minutes after the first deferral the activity signals are ignored. A failed + newest-message read still defers, and that deferral consumes the attempt. + """ + newest = _newest_message(delivery) + first = parse_delivery_timestamp(record.get('first_deferred_at')) + if first is not None and (now - first).total_seconds() >= DEFER_MAX_SECONDS: + return newest + if delivery.services.stream_activity_probe(delivery.user_id, delivery.conversation_id, now): + raise _Defer(False) + if isinstance(newest, dict) and newest.get('role') == 'user': + sent = parse_delivery_timestamp(newest.get('timestamp')) + if sent is None or (now - sent).total_seconds() < RECENT_USER_MESSAGE_SECONDS: + raise _Defer(False) + return newest + + +class _Undeliverable(Exception): + """Post nothing for this generation. One notice says why, unless the reason is silent.""" + + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +def _mark_unread(delivery): + """Mark the chat unread for M before M exists (D9 step 6); the mark and T are one write. + + A retry after ``publishing`` passes the earlier planned time, so a chat that already reached + it (marked then read, or moved on) is left alone. + """ + services = delivery.services + record = delivery.record + message_id = delivery.message_id() + previous = record.get('planned_at') if record.get('phase') == PHASE_PUBLISHING else None + planned_at = _message_timestamp(delivery.now()) + + def publishing(current, _now): + current.update({'phase': PHASE_PUBLISHING, 'message_id': message_id, 'planned_at': planned_at}) + return current + + _write_claimed(delivery, publishing) + try: + outcome, _conversation = services.mark_unread_guarded( + services.conversations, delivery.conversation_id, delivery.user_id, message_id, planned_at, + require_private=True, skip_if_read_since=previous if isinstance(previous, str) and previous else None, + ) + except Exception as exc: + delivery_log( + 'Unread mark failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('unread_unavailable') from None + if outcome in {UNREAD_NOT_FOUND, UNREAD_UNAVAILABLE}: + # The chat stopped taking deliveries between the re-check and the mark. + raise _Undeliverable(REASON_CHAT_UNAVAILABLE) + if outcome not in {UNREAD_MARKED, UNREAD_ALREADY_MARKED, UNREAD_READ_SINCE}: + raise _Transient('unread_conflict') + if outcome == UNREAD_MARKED: + try: + services.bump_cache_version(delivery.user_id, reason='workflow_chat_delivery') + except Exception as exc: + # A stale chat list only delays the unread dot; it never fails the delivery. + delivery_log( + 'Chat cache bump failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + def marked(current, _now): + current['phase'] = PHASE_UNREAD_MARKED + return current + + _write_claimed(delivery, marked) + + +def _create_message(delivery, build_message, *, reason=None): + """Place M after the chat's newest message and create it (D9 step 7). A 409 means it exists. + + ``reason`` is the closed code a note stands in for, kept on the record with the phase. + """ + services = delivery.services + message_id = delivery.message_id() + newest = _newest_message(delivery) + now = delivery.now() + newest_at = parse_delivery_timestamp(newest.get('timestamp')) if newest else None + moment = newest_at + timedelta(microseconds=1) if newest_at is not None and newest_at >= now else now + previous_thread = (newest.get('thread_info') or {}) if newest else {} + thread_info = { + 'thread_id': delivery_thread_id(message_id), + 'previous_thread_id': previous_thread.get('thread_id') if isinstance(previous_thread, dict) else None, + 'active_thread': True, + 'thread_attempt': 1, + } + document = build_message(_message_timestamp(moment), thread_info) + try: + services.messages.create_item(body=document) + except Exception as exc: + if _status_code(exc) != 409: + delivery_log( + 'Delivery message create failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('message_create_failed') from None + + def created(current, _now): + current.update({'phase': PHASE_MESSAGE_CREATED, 'message_id': message_id, 'outcome_reason': reason}) + return current + + _write_claimed(delivery, created) + + +class _TooLarge(Exception): + """Not even the smallest excerpt fits the model's context window.""" + + +class _ResultClosed(Exception): + """The reader closed the result for a reason a retry won't change.""" + + def __init__(self, code): + super().__init__(code) + self.code = code + + +_RESULT_CODE_PREFIX = 'workflow_result_' +_RESULT_RETRY_CODES = frozenset({ + 'workflow_result_storage_unavailable', + 'workflow_result_in_progress', + 'workflow_result_not_finished', +}) +_RESULT_MISSING_CODE = 'workflow_result_not_found' + + +@dataclass +class _Content: + """What this generation posts: the final text, already checked, and its metadata.""" + + kind: str + text: str + reason: Optional[str] = None + descriptor: Optional[dict] = None + contexts: Optional[list] = None + model_deployment_name: Optional[str] = None + token_usage: Optional[dict] = None + check: Any = None + blocked_check: Any = None + + +def _note(delivery, label, kind, *, reason=None, reason_code=None, descriptor=None, contexts=None): + body = delivery_note_text(kind, delivery.workflow_name(), reason_code=reason_code) + return _Content(kind, assemble_delivery_content(label, body), reason=reason, descriptor=descriptor, contexts=contexts) + + +def _read_result(delivery, budget, *, expected_sha256=None): + """Read the run's result at one excerpt budget, mapping the reader's closed reasons (D10).""" + services = delivery.services + try: + return services.read_workflow_result( + delivery.user_id, delivery.workflow_id, delivery.run_id, + expected_sha256=expected_sha256, include_excerpts=True, excerpt_budget_bytes=budget, + ) + except services.result_unavailable_error as exc: + code = getattr(exc, 'code', None) + code = code if isinstance(code, str) and code else 'workflow_result_invalid' + if code in _RESULT_RETRY_CODES: + # Storage was unavailable, or the run document hasn't caught up with the runtime yet. + raise _Transient(code[len(_RESULT_CODE_PREFIX):]) from None + raise _ResultClosed(code) from None + except Exception as exc: + delivery_log( + 'Workflow result read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('result_unavailable') from None + + +def _first_excerpt_text(result): + for item in result.get('excerpts') or []: + if isinstance(item, dict) and isinstance(item.get('text'), str) and item['text'].strip(): + return item['text'] + return '' + + +class _Excerpt: + """The result as last read, and the smaller excerpt budgets left to try (D10).""" + + def __init__(self, delivery, result): + self.delivery = delivery + self.result = result + steps = [step for step in delivery.services.excerpt_budget_steps if type(step) is int] + self.steps = [step for step in steps if 0 < step < EXCERPT_BUDGET_BYTES] + + def step_down(self): + if not self.steps: + raise _TooLarge() + descriptor = self.result.get('descriptor') if isinstance(self.result, dict) else None + expected = descriptor.get('result_sha256') if isinstance(descriptor, dict) else None + # Bound to the digest first read, so a smaller excerpt is always of the same result. + self.result = _read_result(self.delivery, self.steps.pop(0), expected_sha256=expected) + + +def _question(delivery): + """The request's own text, clipped; the fixed question when it's gone or was retracted.""" + services = delivery.services + message_id = delivery.invocation.get('user_message_id') + if not isinstance(message_id, str) or not message_id: + return FALLBACK_QUESTION + try: + message = services.messages.read_item(item=message_id, partition_key=delivery.conversation_id) + except Exception as exc: + if _status_code(exc) == 404: + return FALLBACK_QUESTION + delivery_log( + 'Request message read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('message_unavailable') from None + if ( + not isinstance(message, dict) or message.get('role') != 'user' + or message.get('conversation_id') != delivery.conversation_id + or services.is_message_retracted(message) + ): + return FALLBACK_QUESTION + text = message.get('content') + text = text.strip() if isinstance(text, str) else '' + return text[:QUESTION_MAX_CHARS] if text else FALLBACK_QUESTION + + +def _model_metadata(model): + try: + metadata = model.metadata() + except Exception: + metadata = None + return metadata if isinstance(metadata, dict) else {} + + +def _close_model(delivery, model): + close = getattr(model, 'close', None) + if not callable(close): + return + try: + close() + except Exception as exc: + delivery_log( + 'Delivery model client close failed.', level=logging.INFO, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + +def _resolve_model(delivery, seeds): + """The model for ``seeds``, re-authorized for the requester now; None when it's unavailable.""" + services = delivery.services + try: + identity = services.capture_identity(delivery.user_id, conversation_id=delivery.conversation_id) + return services.resolve_model( + delivery.settings, user_id=delivery.user_id, seeds=seeds, identity_context=identity, + ) + except Exception as exc: + level = logging.INFO if isinstance(exc, services.model_error) else logging.WARNING + delivery_log( + 'Delivery model unavailable.', level=level, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return None + + +def _usage_from(response): + usage = getattr(response, 'usage', None) + values = {} + for field in ('prompt_tokens', 'completion_tokens', 'total_tokens'): + value = getattr(usage, field, None) if usage is not None else None + if type(value) is int and value >= 0: + values[field] = value + return values + + +def _reply_text(services, response): + """The reply, clipped; None for an empty, filtered or refused completion.""" + choices = getattr(response, 'choices', None) or [] + if not choices: + return None + choice = choices[0] + if getattr(choice, 'finish_reason', None) == 'content_filter': + return None + if getattr(getattr(choice, 'message', None), 'refusal', None): + return None + text = services.extract_text(response) + text = text.strip() if isinstance(text, str) else '' + return text[:REPLY_MAX_CHARS] if text else None + + +def _fits(services, model, messages): + """Whether the messages fit the model's window with room for the reply.""" + name = getattr(model, 'model_metadata', None) or getattr(model, 'behavior_name', None) or model.deployment + output_tokens = COMPOSE_MAX_TOKENS + if callable(services.compose_output_tokens): + output_tokens = services.compose_output_tokens(model) + budget = services.calculate_budget( + messages, name, provider=getattr(model, 'provider', None), output_tokens=output_tokens, + ) + return isinstance(budget, dict) and budget.get('decision') == 'full_input' + + +def _compose_with(delivery, model, excerpt, question, time_zone): + """One model's reply, stepping the excerpt down until it fits; None when it can't answer. + + Raises ``_TooLarge`` once the smallest excerpt still doesn't fit. + """ + services = delivery.services + while True: + try: + # A fresh nonce for every request, named by the system prompt that fences the excerpt. + messages = services.build_result_messages( + delivery.settings, excerpt.result, [], question, time_zone, nonce=services.new_fence_nonce(), + ) + fits = _fits(services, model, messages) + except Exception as exc: + delivery_log( + 'Delivery prompt could not be prepared for this model.', level=logging.WARNING, + run_id=delivery.run_id, error_type=type(exc).__name__, + ) + return None + if fits: + break + excerpt.step_down() + try: + response = model.create_completion( + messages=messages, temperature=COMPOSE_TEMPERATURE, max_tokens=COMPOSE_MAX_TOKENS, + ) + reply = _reply_text(services, response) + except Exception as exc: + delivery_log( + 'Delivery compose failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return None + if reply is None: + delivery_log('Delivery compose returned no usable reply.', level=logging.INFO, run_id=delivery.run_id) + return None + return reply, _usage_from(response) + + +def _compose(delivery, excerpt, question, time_zone): + """The reply from the request's model, then the default model (D10). + + Returns ``(reply, deployment, usage)``, or None when neither model answered. + """ + selection = delivery.record.get('model_selection') + chain = [] + if ( + isinstance(selection, dict) and isinstance(selection.get('model'), dict) + and any(selection['model'].values()) + ): + chain.append(selection) + chain.append({}) + tried = set() + for seeds in chain: + model = _resolve_model(delivery, seeds) + if model is None: + continue + try: + metadata = _model_metadata(model) + binding = ( + metadata.get('model_deployment_name'), metadata.get('model_provider'), + metadata.get('model_endpoint_id'), metadata.get('model_id'), + ) + if binding in tried: + continue + tried.add(binding) + composed = _compose_with(delivery, model, excerpt, question, time_zone) + if composed is not None: + reply, usage = composed + return reply, metadata.get('model_deployment_name') or getattr(model, 'deployment', None), usage + finally: + _close_model(delivery, model) + return None + + +def _log_compose_usage(delivery, deployment, usage): + """Count the compose the way 6a counts a Follow up answer (A4). Logging never fails delivery.""" + total = usage.get('total_tokens') if isinstance(usage, dict) else None + if not total: + return + message_id = delivery.message_id() + try: + delivery.services.log_token_usage( + user_id=delivery.user_id, token_type='chat', conversation_id=delivery.conversation_id, + message_id=message_id, model=deployment, workspace_type='personal', total_tokens=total, + prompt_tokens=usage.get('prompt_tokens'), completion_tokens=usage.get('completion_tokens'), + additional_context={'workflow_result_delivery': True}, + idempotency_key=token_usage_idempotency_key(message_id, delivery.record.get('attempts')), + ) + except Exception as exc: + delivery_log( + 'Delivery token usage logging failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + +def _fenced(text): + """A code block whose fence is longer than any run of backticks in the text.""" + longest = max((len(run) for run in re.findall(r'`+', text)), default=0) + fence = '`' * max(3, longest + 1) + return f'{fence}\n{text}\n{fence}' + + +def _disclosure(delivery, result, time_zone, *, truncated=False): + return delivery.services.format_disclosure( + result.get('descriptor') or {}, time_zone, + truncated=truncated or bool(result.get('truncated')), partial=bool(result.get('partial')), + analysis_only=bool(result.get('analysis_only')), skipped_reports=bool(result.get('skipped_reports')), + ) + + +def _checked(delivery, label, content): + """Check the final text as chat output; a blocked reply posts the fixed note instead (D10).""" + services = delivery.services + try: + check = services.check_content( + content.text, 'chat_output', user_id=delivery.user_id, settings=delivery.settings, + ) + except Exception as exc: + delivery_log( + 'Delivery content check failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('content_check_unavailable') from None + if getattr(check, 'blocked', False): + blocked = _note(delivery, label, KIND_CONTENT_BLOCKED, reason=REASON_CONTENT_BLOCKED) + blocked.blocked_check = check + return blocked + content.check = check + return content + + +def _closed_result(delivery, label, code): + """A result the reader closed: the status note, or no post at all when the run is gone.""" + if code == _RESULT_MISSING_CODE: + if _read_workflow(delivery) is None: + raise _Undeliverable(REASON_WORKFLOW_DELETED) + raise _Undeliverable(REASON_ACCESS_LOST) + delivery_log('Workflow result closed; posting the status note.', level=logging.INFO, run_id=delivery.run_id, code=code) + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + + +def _result_content(delivery, label, time_zone): + """The composed reply for a completed run, or the note that stands in for it (D10).""" + services = delivery.services + if not services.is_chat_workflow_results_enabled(delivery.settings, user_roles=delivery.record.get('requester_roles')): + # Without Use Workflow Results In Chat, nothing of the result reaches a model or the chat. + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULTS_OFF) + try: + result = _read_result(delivery, EXCERPT_BUDGET_BYTES) + descriptor = dict(result.get('descriptor') or {}) + contexts = [services.workflow_result_context(descriptor)] + if result.get('saved_inputs'): + return _note(delivery, label, KIND_ANALYSIS, descriptor=descriptor, contexts=contexts) + if not _first_excerpt_text(result): + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + excerpt = _Excerpt(delivery, result) + composed = _compose(delivery, excerpt, _question(delivery), time_zone) + except _ResultClosed as exc: + return _closed_result(delivery, label, exc.code) + except _TooLarge: + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + except services.result_unavailable_error: + # The descriptor didn't name a valid result context, so Follow up couldn't use it. + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + result = excerpt.result + if composed is None: + # Neither model answered: the start of the result, fenced, under a fixed sentence. + body = f'{TRIMMED_FALLBACK_INTRO}\n\n{_fenced(_first_excerpt_text(result)[:TRIMMED_FALLBACK_CHARS])}' + text = assemble_delivery_content(label, body, _disclosure(delivery, result, time_zone, truncated=True)) + content = _Content(KIND_RESULT, text, descriptor=descriptor, contexts=contexts) + else: + reply, deployment, usage = composed + _log_compose_usage(delivery, deployment, usage) + text = assemble_delivery_content(label, reply, _disclosure(delivery, result, time_zone)) + content = _Content( + KIND_RESULT, text, descriptor=descriptor, contexts=contexts, + model_deployment_name=deployment, token_usage=usage, + ) + return _checked(delivery, label, content) + + +def _failure_code(delivery): + """The closed reason code for a failed run: the runtime's gate reason, else its state.""" + try: + summary = _read_control_summary(delivery.services, delivery.user_id, delivery.run) + except _Transient: + summary = {} + return summary.get('gate_reason_code') or delivery.record.get('run_status') + + +def _content(delivery, label, time_zone): + kind = delivery.record.get('kind') + if kind == KIND_RESULT: + return _result_content(delivery, label, time_zone) + if kind == KIND_FAILED: + return _note(delivery, label, KIND_FAILED, reason_code=_failure_code(delivery)) + if kind in {KIND_CANCELLED, KIND_SKIPPED}: + return _note(delivery, label, kind) + raise _Transient('unsupported_kind') + + +def _message_builder(delivery, content): + """Build the delivery message for the timestamp and thread placement chosen at create time.""" + services = delivery.services + record = delivery.record + invocation = delivery.invocation + message_id = delivery.message_id() + + def build(timestamp, thread_info): + metadata = { + DELIVERY_METADATA_KEY: build_delivery_metadata( + kind=content.kind, workflow_id=delivery.workflow_id, run_id=delivery.run_id, + generation=record.get('generation'), run_status=record.get('run_status'), + orchestration_run_id=invocation.get('orchestration_run_id'), step_id=invocation.get('step_id'), + requested_at=invocation.get('requested_at'), + ), + 'token_usage': dict(content.token_usage or {}), + 'user_info': {'user_id': delivery.user_id}, + 'thread_info': thread_info, + } + if content.kind in RESULT_CONTEXT_KINDS and content.descriptor and content.contexts: + # The same lineage keys a 6a answer carries, so Follow up and masking treat it alike. + metadata['workflow_result'] = dict(content.descriptor) + metadata['workflow_result_contexts'] = [dict(item) for item in content.contexts] + document = { + 'id': message_id, + 'conversation_id': delivery.conversation_id, + 'role': 'assistant', + 'content': content.text, + 'timestamp': timestamp, + 'model_deployment_name': content.model_deployment_name, + 'augmented': False, + 'hybrid_citations': [], + 'hybridsearch_query': None, + 'agent_citations': [], + 'web_search_citations': [], + 'user_message': None, + 'metadata': metadata, + } + if content.check is not None: + services.attach_chat_check(document, content.check) + return document + + return build + + +def _record_incident(delivery, check): + """Best-effort incident for a blocked reply. The record holds categories, never text.""" + services = delivery.services + try: + stub = services.attach_chat_check( + {'id': delivery.message_id(), 'conversation_id': delivery.conversation_id}, check, + ) + services.record_incident(stub, delivery.user_id) + except Exception as exc: + delivery_log( + 'Delivery content incident failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + +SWEEP_QUERY = ( + 'SELECT TOP @top c.id, c.user_id FROM c ' + 'WHERE IS_DEFINED(c.chat_delivery) ' + 'AND (NOT IS_DEFINED(c.chat_delivery.next_attempt_at) OR IS_NULL(c.chat_delivery.next_attempt_at) ' + 'OR c.chat_delivery.next_attempt_at <= @now) ' + 'AND (c.chat_delivery.status = @ready ' + 'OR (c.chat_delivery.status = @delivering AND c.chat_delivery.lease_expires_at <= @now) ' + 'OR (c.chat_delivery.status = @pending AND (c.chat_delivery.expires_at <= @now ' + 'OR (ARRAY_CONTAINS(@terminal, c.status) AND c._ts <= @grace_ts))))' +) + +_OUTCOME_BY_STATUS = { + STATUS_DELIVERED: OUTCOME_DELIVERED, + STATUS_UNDELIVERABLE: OUTCOME_UNDELIVERABLE, + STATUS_EXPIRED: OUTCOME_EXPIRED, +} + + +def _write_phase(delivery, phase): + def advance(current, _now): + current['phase'] = phase + return current + + _write_claimed(delivery, advance) + + +def _signal(delivery): + try: + delivery.services.signal_delivery(delivery.user_id, delivery.run_id) + except Exception as exc: + delivery_log('Delivery hint failed.', level=logging.INFO, run_id=delivery.run_id, error_type=type(exc).__name__) + + +def _finish(delivery, status, notice_kind, *, reason=None): + """Close the generation and release the lease. + + The write is reconciled with a fresh control read, so a resume that landed while this + generation was being delivered reopens the record for the next one. + """ + try: + summary = _read_control_summary(delivery.services, delivery.user_id, delivery.run) + except _Transient: + # The projection of the resume's own terminal write reconciles the record instead. + summary = None + + def close(current, now): + current.update({ + 'status': status, + 'notice_kind': notice_kind, + 'next_attempt_at': None, + 'first_deferred_at': None, + }) + if status == STATUS_DELIVERED: + current['delivered_at'] = current.get('delivered_at') or format_delivery_timestamp(now) + if reason is not None: + current['outcome_reason'] = reason + return current + + record = _write_claimed(delivery, close, release=True, summary=summary) + if record.get('status') == STATUS_READY: + _signal(delivery) + delivery_log( + 'Delivery closed.', run_id=delivery.run_id, conversation_id=delivery.conversation_id, + status=status, reason=reason, + ) + return _OUTCOME_BY_STATUS.get(status, OUTCOME_NOT_APPLICABLE) + + +def _close_silently(delivery, reason): + """Close without a post or a notice: the run itself is gone (A1).""" + + def close(current, now): + return _close_for_runtime(current, reason, format_delivery_timestamp(now)) + + _write_claimed(delivery, close, release=True) + delivery_log('Delivery closed silently.', run_id=delivery.run_id, reason=reason) + return OUTCOME_CLOSED_SILENTLY + + +def _conversation_title(delivery): + conversation = delivery.conversation + if conversation is None and isinstance(delivery.conversation_id, str) and delivery.conversation_id: + try: + conversation = delivery.services.conversations.read_item( + item=delivery.conversation_id, partition_key=delivery.conversation_id, + ) + except Exception as exc: + delivery_log( + 'Chat title read failed.', level=logging.INFO, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + conversation = None + if not isinstance(conversation, dict) or conversation.get('user_id') != delivery.user_id: + return '' + title = conversation.get('title') + return title if isinstance(title, str) else '' + + +def _send_chat_notice(delivery): + """The bell notice for a posted message. It opens the chat; its key makes it one per generation.""" + try: + delivery.services.create_chat_response_notification( + delivery.user_id, delivery.conversation_id, delivery.message_id(), + conversation_title=_conversation_title(delivery), + response_preview=delivered_notice_preview(delivery.workflow_name()), + idempotency_key=delivery_notification_key(delivery.run_id, delivery.generation), + strict=True, + ) + except Exception as exc: + delivery_log( + 'Delivery notice failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('notification_failed') from None + + +def _send_notice(delivery, kind, key, message, delivery_status): + """One fixed workflow notice that links to the run. Nothing in it comes from the result.""" + name = delivery.workflow_name() + try: + title = notice_title(kind, name) + except ValueError: + title = notice_title(KIND_STATUS, name) + try: + delivery.services.create_notification( + user_id=delivery.user_id, + notification_type=NOTIFICATION_TYPE, + title=title, + message=message, + link_url=workflow_run_notice_link(delivery.workflow_id, delivery.run_id), + metadata=notice_metadata(delivery.workflow_id, delivery.run_id, delivery_status), + idempotency_key=key, + strict=True, + ) + except Exception as exc: + delivery_log( + 'Delivery notice failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('notification_failed') from None + + +def _notify_delivered(delivery): + if not phase_at_least(delivery.record.get('phase'), PHASE_NOTIFIED): + _send_chat_notice(delivery) + _write_phase(delivery, PHASE_NOTIFIED) + return _finish(delivery, STATUS_DELIVERED, NOTICE_CHAT_RESPONSE) + + +def _undeliverable(delivery, reason): + """Post nothing; send one notice that says the chat can't show the run, unless it's silent. + + The notice's key makes it one per generation, so this path never records ``notified``: that + phase means a message exists. + """ + if reason in SILENT_REASONS: + return _close_silently(delivery, reason) + if phase_at_least(delivery.record.get('phase'), PHASE_MESSAGE_CREATED): + return _notify_delivered(delivery) + _send_notice( + delivery, delivery.record.get('kind'), + undeliverable_notification_key(delivery.run_id, delivery.generation), + UNDELIVERABLE_NOTICE_MESSAGE, STATUS_UNDELIVERABLE, + ) + return _finish(delivery, STATUS_UNDELIVERABLE, NOTICE_UNDELIVERABLE, reason=reason) + + +def _expire(delivery): + """The run didn't finish before the delivery window closed: one notice, nothing in the chat.""" + _send_notice( + delivery, KIND_EXPIRED, expired_notification_key(delivery.run_id), + EXPIRED_NOTICE_MESSAGE, STATUS_EXPIRED, + ) + return _finish(delivery, STATUS_EXPIRED, NOTICE_EXPIRED) + + +def _exhausted(delivery, reason): + """Attempts ran out. Close with a best-effort notice; never post a second message.""" + phase = delivery.record.get('phase') + created = phase_at_least(phase, PHASE_MESSAGE_CREATED) + if not created: + try: + created = _message_exists(delivery, delivery.message_id()) + except _Transient: + # Unknown: an undeliverable notice beats a silent loss. + created = False + if created: + try: + _send_chat_notice(delivery) + notice = NOTICE_CHAT_RESPONSE + except _Transient: + notice = NOTICE_NONE + return _finish(delivery, STATUS_DELIVERED, notice) + if delivery.record.get('kind') == KIND_EXPIRED: + # An expiry whose notice kept failing still closes as an expiry. + try: + _send_notice( + delivery, KIND_EXPIRED, expired_notification_key(delivery.run_id), + EXPIRED_NOTICE_MESSAGE, STATUS_EXPIRED, + ) + notice = NOTICE_EXPIRED + except _Transient: + notice = NOTICE_NONE + return _finish(delivery, STATUS_EXPIRED, notice) + try: + _send_notice( + delivery, delivery.record.get('kind'), + undeliverable_notification_key(delivery.run_id, delivery.generation), + UNDELIVERABLE_NOTICE_MESSAGE, STATUS_UNDELIVERABLE, + ) + notice = NOTICE_UNDELIVERABLE + except _Transient: + notice = NOTICE_NONE + return _finish(delivery, STATUS_UNDELIVERABLE, notice, reason=reason) + + +def _retry_later(delivery, reason): + attempts = _count(delivery.record.get('attempts')) + if attempts >= MAX_ATTEMPTS: + return _exhausted(delivery, REASON_DELIVERY_FAILED) + delay = next_backoff_seconds(attempts) + + def later(current, now): + current.update({'status': STATUS_READY, 'next_attempt_at': _later(now, delay)}) + return current + + _write_claimed(delivery, later, release=True) + delivery_log( + 'Delivery attempt failed; it will be retried.', level=logging.WARNING, run_id=delivery.run_id, + reason=reason, attempt=attempts, + ) + return OUTCOME_RETRY_SCHEDULED + + +def _defer(delivery, counted): + """Wait for the chat to settle (D7). An uncounted deferral gives the claim's attempt back. + + A counted deferral (the chat's activity couldn't be read) backs off like any failed attempt. + """ + attempts = _count(delivery.record.get('attempts')) + if counted and attempts >= MAX_ATTEMPTS: + return _exhausted(delivery, REASON_DELIVERY_FAILED) + delay = next_backoff_seconds(attempts) if counted else DEFER_RETRY_SECONDS + + def deferred(current, now): + current.update({'status': STATUS_READY, 'next_attempt_at': _later(now, delay)}) + if not counted: + current['attempts'] = max(0, _count(current.get('attempts')) - 1) + if not current.get('first_deferred_at'): + current['first_deferred_at'] = format_delivery_timestamp(now) + return current + + _write_claimed(delivery, deferred, release=True) + delivery_log('Delivery deferred while the chat is busy.', run_id=delivery.run_id, step='counted' if counted else 'activity') + return OUTCOME_DEFERRED + + +def _asked_when(delivery, time_zone): + try: + asked = delivery.services.format_run_time(delivery.invocation.get('requested_at'), time_zone) + except Exception as exc: + delivery_log('Request time could not be formatted.', level=logging.INFO, run_id=delivery.run_id, error_type=type(exc).__name__) + return '' + return asked if isinstance(asked, str) else '' + + +def _deliver(delivery): + """One claimed attempt at the generation, resuming from the phase the last attempt reached (D9).""" + services = delivery.services + if delivery.exhausted: + return _exhausted(delivery, REASON_DELIVERY_FAILED) + try: + delivery.settings = services.get_settings() or {} + except Exception as exc: + delivery_log('Settings read failed.', level=logging.WARNING, run_id=delivery.run_id, error_type=type(exc).__name__) + raise _Transient('settings_unavailable') from None + delivery.workflow = _read_workflow(delivery) + if delivery.workflow is None: + return _close_silently(delivery, REASON_WORKFLOW_DELETED) + + phase = delivery.record.get('phase') + if not phase_at_least(phase, PHASE_MESSAGE_CREATED) and _message_exists(delivery, delivery.message_id()): + # Check before compose (D9 step 4): an earlier attempt at this generation already posted M, + # whether its phase write was lost or the record itself was reverted. Skip compose and the + # unread mark; the notice below is idempotent by its key. + _write_phase(delivery, PHASE_MESSAGE_CREATED) + if phase_at_least(delivery.record.get('phase'), PHASE_MESSAGE_CREATED): + return _notify_delivered(delivery) + record = delivery.record + if record.get('kind') == KIND_EXPIRED: + return _expire(delivery) + + if _read_conversation(delivery) is None: + raise _Undeliverable(REASON_CHAT_UNAVAILABLE) + if not services.is_user_workflows_enabled(delivery.settings, user_roles=record.get('requester_roles')): + raise _Undeliverable(REASON_ACCESS_LOST) + now = delivery.now() + if not phase_at_least(record.get('phase'), PHASE_PUBLISHING): + expires_at = parse_delivery_timestamp(record.get('expires_at')) + if expires_at is not None and now > expires_at: + raise _Undeliverable(REASON_EXPIRED_BEFORE_DELIVERY) + _defer_probe(delivery, record, now) + + time_zone = record.get('time_zone') + label = delivery_label(delivery.workflow_name(), _asked_when(delivery, time_zone)) + content = _content(delivery, label, time_zone) + if not phase_at_least(delivery.record.get('phase'), PHASE_UNREAD_MARKED): + _mark_unread(delivery) + _create_message(delivery, _message_builder(delivery, content), reason=content.reason) + if content.blocked_check is not None: + _record_incident(delivery, content.blocked_check) + return _notify_delivered(delivery) + + +def _handle(delivery, handler, *args): + """Record the attempt's outcome. If even that fails, the lease lapses and a later claim retries.""" + try: + try: + return handler(delivery, *args) + except _Transient as exc: + if handler is not _undeliverable: + raise + return _retry_later(delivery, exc.reason) + except _LostLease: + return OUTCOME_LOST_LEASE + except Exception as exc: + delivery_log( + 'Delivery outcome could not be recorded.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return OUTCOME_UNAVAILABLE + + +def _run_claimed(delivery): + try: + return _deliver(delivery) + except _LostLease: + return OUTCOME_LOST_LEASE + except _Undeliverable as exc: + return _handle(delivery, _undeliverable, exc.reason) + except _Defer as exc: + return _handle(delivery, _defer, exc.counted) + except _Transient as exc: + return _handle(delivery, _retry_later, exc.reason) + except Exception as exc: + delivery_log( + 'Delivery attempt failed unexpectedly.', level=logging.ERROR, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return _handle(delivery, _retry_later, 'unexpected_error') + + +def _services_or_default(services): + if services is not None: + return services + try: + return default_workflow_chat_delivery_services() + except Exception as exc: + delivery_log('Delivery services unavailable.', level=logging.ERROR, error_type=type(exc).__name__) + return None + + +def process_workflow_chat_delivery(user_id, run_id, *, services=None): + """Deliver one run's outcome when it's due. Returns an ``OUTCOME_*`` string and never raises.""" + if not isinstance(user_id, str) or not user_id or not isinstance(run_id, str) or not run_id: + return OUTCOME_NOT_APPLICABLE + services = _services_or_default(services) + if services is None: + return OUTCOME_UNAVAILABLE + try: + claimed = _claim(services, user_id, run_id) + except _LostLease: + return OUTCOME_LOST_LEASE + except Exception as exc: + delivery_log('Delivery claim failed.', level=logging.WARNING, run_id=run_id, error_type=type(exc).__name__) + return OUTCOME_UNAVAILABLE + if not isinstance(claimed, _Delivery): + return claimed + return _run_claimed(claimed) + + +def process_workflow_chat_delivery_hints(services=None, limit=HINT_BATCH): + """Process this process's queued hints. They need no lock: every claim is an ETag write.""" + services = _services_or_default(services) + if services is None: + return [] + try: + hints = list(services.drain_hints(limit)) + except Exception as exc: + delivery_log('Delivery hints could not be read.', level=logging.WARNING, error_type=type(exc).__name__) + return [] + outcomes = [] + for user_id, run_id in hints: + outcomes.append((run_id, process_workflow_chat_delivery(user_id, run_id, services=services))) + return outcomes + + +def run_workflow_chat_delivery_sweep(services=None): + """Find due deliveries across all users and process a bounded batch under the sweep lock (D8). + + Returns ``{'locked', 'processed', 'outcomes'}``. Without the lock nothing is queried. + """ + summary = {'locked': False, 'processed': 0, 'outcomes': {}} + services = _services_or_default(services) + if services is None: + return summary + try: + lock = services.acquire_lock(SWEEP_LOCK_NAME, SWEEP_LOCK_SECONDS) + except Exception as exc: + delivery_log('Delivery sweep lock failed.', level=logging.WARNING, error_type=type(exc).__name__) + return summary + if not lock: + return summary + summary['locked'] = True + started = services.monotonic() + try: + now = _now(services) + rows = services.runs.query_items( + query=SWEEP_QUERY, + parameters=[ + {'name': '@top', 'value': SWEEP_TOP}, + {'name': '@now', 'value': format_delivery_timestamp(now)}, + {'name': '@ready', 'value': STATUS_READY}, + {'name': '@delivering', 'value': STATUS_DELIVERING}, + {'name': '@pending', 'value': STATUS_PENDING}, + {'name': '@terminal', 'value': sorted(RUN_DOCUMENT_TERMINAL_STATUSES)}, + {'name': '@grace_ts', 'value': int(now.timestamp()) - SWEEP_TS_GRACE_SECONDS}, + ], + enable_cross_partition_query=True, + ) + for row in rows: + if summary['processed'] >= SWEEP_MAX_RUNS or services.monotonic() - started >= SWEEP_MAX_SECONDS: + break + if not isinstance(row, dict): + continue + outcome = process_workflow_chat_delivery(row.get('user_id'), row.get('id'), services=services) + summary['processed'] += 1 + summary['outcomes'][outcome] = summary['outcomes'].get(outcome, 0) + 1 + except Exception as exc: + delivery_log('Delivery sweep failed.', level=logging.WARNING, error_type=type(exc).__name__) + finally: + try: + services.release_lock(lock) + except Exception as exc: + delivery_log('Delivery sweep lock release failed.', level=logging.WARNING, error_type=type(exc).__name__) + return summary + + +__all__ = ( + 'HINT_BATCH', + 'OUTCOME_BUSY', + 'OUTCOME_CLOSED_SILENTLY', + 'OUTCOME_DEFERRED', + 'OUTCOME_DELIVERED', + 'OUTCOME_EXPIRED', + 'OUTCOME_LOST_LEASE', + 'OUTCOME_NOT_APPLICABLE', + 'OUTCOME_NOT_DUE', + 'OUTCOME_PENDING', + 'OUTCOME_RETRY_SCHEDULED', + 'OUTCOME_UNAVAILABLE', + 'OUTCOME_UNDELIVERABLE', + 'QUESTION_MAX_CHARS', + 'SWEEP_QUERY', + 'WorkflowChatDeliveryServices', + 'default_workflow_chat_delivery_services', + 'process_workflow_chat_delivery', + 'process_workflow_chat_delivery_hints', + 'run_workflow_chat_delivery_sweep', +) diff --git a/application/single_app/functions_workflow_runtime.py b/application/single_app/functions_workflow_runtime.py index 74b6c21b1..7a38920c7 100644 --- a/application/single_app/functions_workflow_runtime.py +++ b/application/single_app/functions_workflow_runtime.py @@ -16,6 +16,10 @@ from functions_appinsights import log_event from functions_m365_workflow_binding import M365_WAITING_STATES +from functions_workflow_chat_delivery import ( + CHAT_DELIVERY_KEY, control_summary, delivery_log, finalize_chat_delivery_seed, reconcile_chat_delivery, + signal_workflow_chat_delivery, +) from functions_workflow_definitions import ( WORKFLOW_DEFINITION_FIELDS, validate_workflow_publication_completion, workflow_definition_revision, ) @@ -149,7 +153,7 @@ def workflow_run_id_for_request(workflow, request_id): def queue_durable_workflow_run( workflow, *, actor_user_id, trigger_source="manual", request_id=None, invocation_metadata=None, - chat_invocation=None, + chat_invocation=None, chat_delivery=None, ): services = _services(workflow) settings = services["settings"]() @@ -227,6 +231,9 @@ def queue_durable_workflow_run( if chat_invocation is not None: # Which chat orchestration step started this run, for the requester's own run history. run["chat_invocation"] = deepcopy(chat_invocation) + if chat_delivery is not None: + # The record that has this run's result posted back to that chat when the run finishes. + run[CHAT_DELIVERY_KEY] = finalize_chat_delivery_seed(chat_delivery, control) try: saved = services["runs"].read_item(item=run_id, partition_key=services["partition"]) except CosmosResourceNotFoundError: @@ -247,6 +254,28 @@ def queue_durable_workflow_run( return {"success": True, "run": saved, "workflow": bound, "runtime": workflow_runtime_projection(control)} +def _reconcile_chat_delivery(body, control, run_id): + """Move a chat-started run's delivery record along with its runtime. Never raises. + + Returns True when the record is ready for its result to be posted back to the chat. A failure + keeps the stored record as it was, and the delivery sweep catches it up later. + """ + record = body.get(CHAT_DELIVERY_KEY) + if not isinstance(record, dict): + return False + try: + reconciled, changed, ready = reconcile_chat_delivery(record, control_summary(control)) + except Exception as exc: + delivery_log( + 'A chat delivery record was not reconciled with its run.', level=logging.WARNING, + run_id=run_id, error_type=type(exc).__name__, + ) + return False + if changed: + body[CHAT_DELIVERY_KEY] = reconciled + return bool(ready) + + def _project_runtime_run(services, workflow, run_id, control, *, result=None, attempt=0): latest = workflow_runtime_store(workflow, run_id).read() if latest["version"] > control["version"]: @@ -261,6 +290,11 @@ def _project_runtime_run(services, workflow, run_id, control, *, result=None, at body = {key: value for key, value in existing.items() if not key.startswith("_")} if result: body.update(result.get("run") or {}) + # The stored chat delivery record wins over any copy a result or checkpoint carried. + if CHAT_DELIVERY_KEY in existing: + body[CHAT_DELIVERY_KEY] = deepcopy(existing[CHAT_DELIVERY_KEY]) + else: + body.pop(CHAT_DELIVERY_KEY, None) body.update( durable_execution=True, status=control["state"], runtime=workflow_runtime_projection(control), @@ -272,6 +306,7 @@ def _project_runtime_run(services, workflow, run_id, control, *, result=None, at if int(existing.get("runtime_version") or 0) > control["version"]: return existing body["runtime_version"] = control["version"] + delivery_ready = _reconcile_chat_delivery(body, control, run_id) try: services["runs"].replace_item( item=run_id, body=body, etag=existing["_etag"], match_condition=MatchConditions.IfNotModified, @@ -285,6 +320,8 @@ def _project_runtime_run(services, workflow, run_id, control, *, result=None, at attempt=attempt + 1, ) raise + if delivery_ready: + signal_workflow_chat_delivery(body.get("user_id"), run_id) for _attempt in range(5): current = services["definitions"].read_item(item=workflow["id"], partition_key=services["partition"]) if current.get("deleting"): diff --git a/application/single_app/route_backend_conversations.py b/application/single_app/route_backend_conversations.py index 1ff97f487..f652808b4 100644 --- a/application/single_app/route_backend_conversations.py +++ b/application/single_app/route_backend_conversations.py @@ -97,6 +97,12 @@ is_saved_analysis_unavailable, sanitize_saved_analysis_messages, ) +from functions_workflow_chat_delivery import ( + DELIVERY_EDIT_UNSUPPORTED, + DELIVERY_RETRY_UNSUPPORTED, + is_workflow_delivery_message, + workflow_delivery_refusal_payload, +) from functions_workflow_result_masking import message_asks_about_workflow_result, message_uses_workflow_result from functions_workflow_result_reader import WorkflowResultUnavailable, workflow_result_error_payload from utils_cache import invalidate_personal_search_cache @@ -3143,6 +3149,11 @@ def retry_message(message_id): return jsonify({'error': 'Conversation not found'}), 404 elif message_user_id != user_id: return jsonify({'error': 'You can only retry your own messages'}), 403 + + if is_workflow_delivery_message(original_msg): + # A workflow run posted this message, so there is no question to replay. + payload, status = workflow_delivery_refusal_payload(DELIVERY_RETRY_UNSUPPORTED) + return jsonify(payload), status # Get thread info from original message thread_id = original_msg.get('metadata', {}).get('thread_info', {}).get('thread_id') @@ -3366,6 +3377,11 @@ def edit_message(message_id): original_msg = message_results[0] conversation_id = original_msg.get('conversation_id') original_role = original_msg.get('role') + + if is_workflow_delivery_message(original_msg): + # A workflow run posted this message, so there is nothing of the user's to edit. + payload, status = workflow_delivery_refusal_payload(DELIVERY_EDIT_UNSUPPORTED) + return jsonify(payload), status # Only allow editing user messages if original_role != 'user': diff --git a/application/single_app/route_backend_orchestration.py b/application/single_app/route_backend_orchestration.py index 5a9cce766..423678a81 100644 --- a/application/single_app/route_backend_orchestration.py +++ b/application/single_app/route_backend_orchestration.py @@ -207,7 +207,7 @@ normalize_elicitation, safe_failure, ) -from functions_settings import get_settings, get_user_settings +from functions_settings import enabled_required, get_settings, get_user_settings, workflow_user_required from functions_model_catalog import ModelCatalogError from functions_orchestration_model_routing import answer_selection, validate_auto_bindings from functions_prompt_metadata import build_prompt_selection_metadata @@ -1457,6 +1457,13 @@ def _workflow_run_links(): return functions_orchestration_workflow_run_links +def _workflow_run_status(): + """The workflow run status module, imported on first use like the run links module.""" + import functions_workflow_chat_delivery_status + + return functions_workflow_chat_delivery_status + + def _proposal_identity(user_id): """The requester's id, email, roles and tenant for a workflow proposal request. @@ -2931,3 +2938,40 @@ def orchestration_workflow_run_links(run_id): ) return jsonify(links.error_payload(links.SERVICE_UNAVAILABLE_CODE)), 503 return jsonify(payload), 200 + + @bp.route("/api/v2/orchestration/workflow-runs/status", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required('allow_user_workflows') + @workflow_user_required + def orchestration_workflow_run_status(): + """Status rows for the saved workflow runs the requester's chats started. + + With one ``conversation_id``, the rows are that chat's runs, newest request first. Without + one, they are the requester's runs still in flight, still waiting to post back, or posted + in the last ten minutes. Each row reads only the requester's own runs and carries fixed + codes and labels, never a run's output or error text. Nothing is written. + """ + user_id = get_current_user_id() + if not user_id: + return jsonify({'error': 'User not authenticated'}), 401 + status = _workflow_run_status() + conversation_values = request.args.getlist('conversation_id') + try: + payload = status.workflow_run_status_payload(user_id, conversation_values) + except status.WorkflowRunStatusError as exc: + body, code = exc.payload() + return jsonify(body), code + except Exception as exc: + conversation_id = conversation_values[0] if len(conversation_values) == 1 else None + log_event( + '[ORCHESTRATION] The workflow run status could not be loaded.', level=logging.ERROR, + extra={ + **workflow_log_context(conversation_id=conversation_id), + 'stage': 'workflow_run_status', 'error_type': type(exc).__name__, + }, + ) + body, code = status.WorkflowRunStatusError(status.STATUS_UNAVAILABLE_CODE).payload() + return jsonify(body), code + return jsonify(payload), 200 diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index 887c8c8d0..25a868e23 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -294,6 +294,13 @@ not claim that research verified the requested details. | Run Workflows From Chat | Lets a plan start one of the user's saved personal workflows when the user asks for it, such as "run my weekly digest now". A plan that starts a workflow always waits for the user to run it, even when the approval mode would run it automatically or after a countdown, and approving it starts each named workflow once. Runs are offered only in the user's own private conversations, only to users who may already create personal workflows, and only for workflows with durable execution on. It is independent of Propose Workflows From Chat, so either can be on without the other. | Off | `enable_chat_orchestration_workflow_runs`; requires Chat Orchestration and personal workflows (`allow_user_workflows`), and the **Run workflows** capability when the Capabilities list is narrowed. Since **0.261.212** | | Capabilities | Restricts which capabilities a plan may use. An empty selection means every capability the other settings already permit. | All | `chat_orchestration_enabled_capabilities` | +When **Run Workflows From Chat** starts a durable personal workflow, the plan +still ends right away. If **Use Workflow Results In Chat** is also on in +Workflow settings, the server records enough context to post the run's eventual +outcome back to the same private chat later, mark it unread and send one bell +notification. If the chat can no longer take the result, the server posts +nothing and sends one workflow notification instead. + ### Actions and agents Use **Use an action** for a question that needs a particular integration, such as a ticket @@ -564,6 +571,7 @@ then report a rejected proposal. | A step says "Your workflow results couldn't be read right now. Try again in a moment." | Workflow storage could not be read when the step ran. The step is retried once automatically. | Once storage is reachable, the user can select **Retry from failed step** or ask again. Reading a result changes nothing, so trying again is safe. | | An answer says a workflow result it used changed or is no longer available, so it was not saved | A run the plan read was deleted or changed, or its owner lost access to a source it used, while the answer was being written. | Expected. Ask again in a new message to read the current result. | | A saved run cannot be resumed | Checkpoints are absent or invalid, relevant context/access changed, or a newer/live attempt exists. | Follow the recovery explanation. Open the current attempt or create a new plan as appropriate; do not infer results from old summaries. | +| A chat-started workflow finishes but no result appears in the chat | The workflow settings gate for result post-back is off, the chat was deleted or shared after the run started, access to personal workflows was lost, the workflow was deleted, or the delivery worker deferred or closed the generation after retryable storage/notification failures. | Check **Use Workflow Results In Chat** under Workflow settings, keep the request in the user's private chat, and open the run from Workflows or the workflow notification. The plan itself is already complete and does not wait for delivery. | ## Related diff --git a/docs/admin/workflow.md b/docs/admin/workflow.md index 2f19607fd..ee48d7d8b 100644 --- a/docs/admin/workflow.md +++ b/docs/admin/workflow.md @@ -65,6 +65,13 @@ visible to its members. Turning one on does not turn on the other. | Workflow Repeat Iteration Limit | Bounds one automatic Repeat until batch, including its first round. Authors must choose an explicit per-block maximum; a new run above this ceiling is rejected rather than shortened. | 25 | `workflow_max_repeat_iterations`; supported range 1-1,000; new runs only; active runs and manual continuation retain the admitted policy | | Workflow Minimum Schedule Interval (seconds) | Sets the shortest fixed interval a personal or group workflow may be saved with, so no one can schedule a workflow to run every few seconds. It's checked only when an interval schedule is created or changed; workflows already saved on a shorter interval keep running. Calendar schedules run at most once a day and are never checked. The default, 1 second, allows every interval, as before. | 1 | `workflow_min_schedule_interval_seconds`; supported range 1-86,400; new or changed interval schedules only | +When **Use Workflow Results In Chat** is on, the same gate also controls whether +a durable personal workflow started from chat can post its terminal outcome back +into that private chat later. The server checks the gate when the chat starts +the run and again before it reads or posts result text, so turning it off leaves +future chat-started runs without a post-back and makes already-started runs post +a fixed status note instead of result text. + The action and task limits apply to personal and group runs alike, so they stay in effect whichever capability is enabled. @@ -274,6 +281,7 @@ See [Workflow publication completion](../explanation/features/WORKFLOW_PUBLICATI | The AI workflow assistant is temporarily unavailable for everyone | The model deployment is throttled or unreachable, or the settings container that holds the rate-limit count can't be reached. The assistant won't run without its limit. | Check the GPT deployment's health and throttling, and Cosmos DB availability. Each request logs `[WorkflowAssist] Assist request finished` with its `status` and error `code`, and no content. | | Users don't see **Ask in chat** or **Ask about this** in V2 | Use Workflow Results In Chat or Enable Personal Workflows is off, or Require WorkflowUser App Role is on and the user doesn't hold the role. Both buttons also need a completed run of a personal workflow, and **Ask in chat** isn't offered on structured runs. | Check both settings, then the `WorkflowUser` role assignment. | | A chat answer reads "This answer is unavailable because access to the workflow result it used could not be confirmed." | The run was deleted, its result changed, its owner can no longer read a source its tasks used, the chat was shared or converted to a collaboration, or storage couldn't be read during the check. | Expected. An answer is shown only while its result still checks out for the chat's owner in a private chat; after a storage failure it shows again once a check succeeds. The original chat's stored messages are unchanged. The collaboration's copies are stored with these answers withheld, and the chat's saved summary is cleared on both chats. Classic chat's message list doesn't run this check, so the owner still sees the answer there. Each withheld answer logs `[WorkflowResults] A chat answer's workflow result was withheld on read.` with a `check` of `conversation` or `result`, a `code` or `error_type`, and no content. | +| Results of a workflow started from chat never appear in the chat | Use Workflow Results In Chat was off when the chat started the run, the requester no longer passes the personal workflow gate, the chat was deleted or shared, the workflow was deleted, the runtime expired, or delivery closed after repeated storage/notification failures. | Check **Use Workflow Results In Chat**, **Enable Personal Workflows**, the `WorkflowUser` role, the chat's privacy, and the run in workflow history. Deleted workflows and missing runtime records close silently; other undeliverable runs create one workflow notification with a run link. | ## Related diff --git a/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md b/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md index fa65b8a20..eadcc1322 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md @@ -25,6 +25,15 @@ starts the run and links to it. It never waits for the run and never reads its results. A plan that starts a workflow always waits for the user to run it, whatever approval mode they chose. +Since version **0.261.227**, when **Use Workflow Results In Chat** is also on, +the server can post a chat-started run's outcome back into the chat that asked +after the run finishes. The started-run note changes from pointing only to run +history to "I'll post the results here when the run finishes. You can also +follow it in the workflow's run history in Workflows." For multiple runs it says +"I'll post each run's results here when it finishes. You can also follow them +in each workflow's run history in Workflows." See +[Workflow result delivery to chat](CHAT_WORKFLOW_RESULT_DELIVERY.md). + What this version adds: - **The `workflow_run` capability.** One plan step per workflow, naming it by a diff --git a/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md b/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md index 6af33bcea..87087cf96 100644 --- a/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md +++ b/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md @@ -728,3 +728,6 @@ the controls are listed in - Re-checking workflow result contexts when a generated file is published. - Phase 6b's delivery, run card, recurring-workflow card and chat-list indicator, and `v2WorkflowRunPath`. +- Phase 6b-1 now has its server-side delivery contract in + [Workflow result delivery to chat](CHAT_WORKFLOW_RESULT_DELIVERY.md); the V2 + run card and chat-list indicator still follow in later work. diff --git a/docs/explanation/features/CHAT_WORKFLOW_RESULT_DELIVERY.md b/docs/explanation/features/CHAT_WORKFLOW_RESULT_DELIVERY.md new file mode 100644 index 000000000..888f183da --- /dev/null +++ b/docs/explanation/features/CHAT_WORKFLOW_RESULT_DELIVERY.md @@ -0,0 +1,869 @@ +# Workflow result delivery to chat + +Implemented in version: **0.261.227**. + +Application version tracking: `application\single_app\config.py`. + +Related issue: #1546 (part 6b-1), part of #1543. Builds on +[Chat orchestration workflow runs](CHAT_ORCHESTRATION_WORKFLOW_RUNS.md) +(#1594), [Workflow results in chat](CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md) +(#1546 part 6a), and the V2 workflow run link described in +[Document provenance](DOCUMENT_PROVENANCE.md) (#1562). The +[Chat orchestration workflows roadmap](CHAT_ORCHESTRATION_WORKFLOWS_ROADMAP.md) +describes all the phases; this is the server-side part of Phase 6b. + +## Overview and dependencies + +A chat orchestration plan can start one of the requester's saved personal +workflows and then finish immediately. Before this version, the user had to open +Workflows later to see what the run produced. With **Use Workflow Results In +Chat** on, the server posts that run's terminal outcome back into the same +private chat that asked for it, even if the run finishes hours later. + +The delivery is server-side. It ships no V2 run-card UI in this pull request. +6b-2 polls the status route documented below and decides how to render cards, +Retry and Open run. + +What this version adds: + +- a `chat_delivery` run record seeded into the run's first durable write when a + chat plan starts a personal durable workflow and the role-aware **Use Workflow + Results In Chat** gate passes. +- runtime projection reconciliation that moves that record from pending to ready + at the terminal durable runtime generation, reopens it only for a later resume + generation, expires old in-flight records, and signals a worker without + blocking the run. +- an idempotent delivery worker that re-checks chat ownership, privacy, + workflow access and the gate; defers while a chat may be mid-stream; composes + a bounded result summary; posts one deterministic message per generation; + marks the chat unread; and sends one bell notification. +- fixed application notes for failures, cancellations, skipped runs, saved + analyses, content-blocked results and results that cannot be shown in chat. +- undeliverable notices when the chat cannot accept the post any more, such as + a deleted chat, a shared chat or lost access. Deleted workflows and missing + runtime controls close silently. +- classic Retry/Edit refusal for any message a workflow run posted. +- an owner-only status route for 6b-2: + `GET /api/v2/orchestration/workflow-runs/status[?conversation_id=...]`. + +Dependencies: + +- Personal workflows (`allow_user_workflows`). When + `require_member_of_workflow_user` is on, the requester also needs the + `WorkflowUser` app role. +- Chat orchestration run steps (`enable_chat_orchestration_workflow_runs`) from + Phase 5. Delivery never starts a workflow by itself. +- Durable execution. Chat-started workflow runs eligible for delivery are the + durable runs started by Phase 5. +- Workflow results in chat (`enable_chat_workflow_results`), off by default. + The role-aware gate `is_chat_workflow_results_enabled_for_user` is checked at + start and again at delivery. +- The background task host and the durable workflow scheduler. The worker uses a + hint queue for prompt delivery and a locked sweep as the backstop. + +## Technical specifications + +### Architecture and flow + +```mermaid +sequenceDiagram + participant Chat as Chat orchestration + participant Runtime as Durable runtime + participant Runs as personal_workflow_runs + participant Worker as Delivery worker + participant Messages as Chat messages + participant Bell as Notifications + + Chat->>Runtime: queue durable run with chat_invocation and optional chat_delivery + Runtime->>Runs: create run document and control + Runtime->>Runs: project terminal control.version onto run + Runtime->>Runs: reconcile chat_delivery to ready(kind, generation) + Runtime-->>Worker: signal hint(user_id, run_id) + Worker->>Runs: ETag claim ready generation + Worker->>Worker: re-check chat, workflow, gates and stream activity + Worker->>Runtime: read bounded workflow result excerpt when kind=result + Worker->>Worker: compose or choose fixed application note + Worker->>Chat: mark conversation unread for deterministic message id + Worker->>Messages: create assistant_workflow_delivery_* message if absent + Worker->>Bell: create one idempotent notice + Worker->>Runs: close delivered, undeliverable or expired +``` + +The plan does not wait for the workflow. Its started-run note explains where the +result will appear. When every run that answer started carries a delivery seed, +the note uses these fixed strings from the contract module: + +- `I'll post the results here when the run finishes. You can also follow it in the workflow's run history in Workflows.` +- `I'll post each run's results here when it finishes. You can also follow them in each workflow's run history in Workflows.` + +Linked or rebuilt runs keep the older Phase 5 note text, because only a fresh +start that recorded this chat's delivery sets the sidecar `chat_delivery: true`. + +### The `chat_delivery` record + +The run document holds a `chat_delivery` object. It contains only IDs, statuses, +model identity fields, roles, timestamps, lease data and closed reason codes. It +never stores result text, prompt text, model output or exception text. + +| Field | Meaning | +| --- | --- | +| `version` | Contract version, currently `1`. | +| `status` | `pending`, `ready`, `delivering`, `delivered`, `undeliverable` or `expired`. | +| `generation` | Durable runtime control `version` for the terminal projection being delivered, or `null` before one is ready. | +| `expires_at` | Delivery deadline. Runtime `deadline_at + 24h` when available, otherwise seed time plus the admitted deadline, capped at 86,400 seconds, plus 24 hours. | +| `time_zone` | Request time zone, or planning time zone, or `UTC`. | +| `model_selection` | Bounded request model identity: `model_deployment`, `model_id`, `model_endpoint_id`, `model_provider`, `reasoning_effort` and up to 10 group ids. | +| `requester_roles` | Up to 20 role names, each up to 64 characters, captured at start for headless gate checks. | +| `lease_id`, `lease_expires_at` | The ETag claim fence. Leases last 300 seconds. | +| `attempts`, `next_attempt_at`, `first_deferred_at` | Retry and chat-settling state. Backoff is 30s, 1m, 2m, 4m, 8m, 15m, 15m, 15m; at 8 attempts the record closes. | +| `phase` | `claimed`, `publishing`, `unread_marked`, `message_created`, `notified` or `null`. | +| `message_id`, `planned_at` | The deterministic message id and the planned unread/message time. | +| `kind` | `result`, `failed`, `cancelled`, `status`, `content_blocked`, `analysis`, `expired` or `skipped`. | +| `notice_kind` | `chat_response`, `undeliverable`, `expired`, `none` or `null`. | +| `outcome_reason` | Closed code such as `chat_unavailable`, `access_lost`, `workflow_deleted`, `runtime_missing`, `delivery_failed`, `expired_before_delivery`, `content_blocked`, `results_off`, `result_unavailable` or `deadline_exceeded`. | +| `run_status` | Runtime state observed for the generation. | +| `delivered_at`, `created_at`, `updated_at` | UTC timestamps written in the delivery format. | +| `history` | Up to five summaries of older generations, with codes and ids only. | + +The origin fields stay in `chat_invocation`: `conversation_id`, +`user_message_id`, `orchestration_run_id`, `attempt_root_run_id`, `step_id`, +`requested_by` and `requested_at`. + +### Generation and reopen rule + +A delivery generation is the durable runtime control `version` at the terminal +projection. The message id, notification idempotency key and status row all use +that generation. + +A `delivered` or `undeliverable` record reopens only when the live control's +`version` is greater than the record's `generation` and the control is not +`deleted`. Durable resume is the normal source of that version bump. The resume +route keeps the same `run_id` and expects the client body +`{expected_version, request_id}`. + +Tombstone deletion is the only non-resume write that bumps a terminal control's +version. It sets `deleted: true`, so it does not reopen delivery. A deleted +control closes a pending or ready record silently as `workflow_deleted`. A +missing control closes within one sweep as `runtime_missing`, also silently. +Expired records never reopen. An already delivered record stays delivered when a +control is deleted later. + +### Guarded save and exactly-once delivery + +There are two layers that keep delivery exactly once: + +1. `save_personal_workflow_run` uses a guarded, merge-preserving ETag path for + runs that carry `chat_delivery`, and for the double-fault case where a + chat-started run lacks `chat_invocation`. The stored `chat_delivery` always + wins over an incoming stale copy. The stored `chat_invocation` and stored + cancellation request are preserved where needed. +2. The worker's side effects are idempotent. It claims with ETag and a lease, + checks for the deterministic message before composing, uses one message id + per `(run_id, conversation_id, generation)`, treats message-create 409 as + success, marks unread under an ETag guard, and sends notifications with + deterministic keys. + +The deterministic message id is: + +```text +assistant_workflow_delivery_{sha256(canonical_json([run_id, conversation_id, generation]))[:40]} +``` + +The thread id is `uuid5(NAMESPACE_URL, message_id)`. + +### Worker phases + +For each due generation the worker: + +1. **Claims and reconciles.** It point-reads the run, reads the runtime control + with deleted controls allowed, reconciles the delivery record, and ETag + replaces the record to `delivering` with a fresh lease and `phase: claimed`. +2. **Skips ahead if the message exists.** After it reads settings and the + workflow, and before any other check, it point-reads the deterministic + message id unless the record already says the message was created. If the + message exists, because a crash lost the phase write or a stale save reverted + the record, it records `message_created` and goes straight to the notice: no + compose, no unread mark and no second message. A failed read retries later. +3. **Re-checks.** It requires the original conversation to exist, belong to the + requester, not be deleted or `orchestration_deleted`, and still be private. + It requires the workflow to exist and not be `deleting`, and it re-checks + personal workflow access from live settings and the stored requester roles. +4. **Defers while the chat may be busy.** It checks shared stream activity, + fresh user messages and newest-message read failures. Soft activity signals + stop delaying after 15 minutes; a failed newest-message read still defers and + consumes an attempt. +5. **Builds content.** Result generations read bounded excerpts and may compose; + failed, cancelled, skipped, status, analysis and content-blocked paths use + fixed application text. +6. **Marks unread before creating the message.** The helper re-reads the + conversation, re-checks owner/privacy/delete state, raises `last_updated` to + the planned message time and writes unread fields with ETag. Retries keep + `skip_if_read_since`, so a user who already read the chat is not re-marked. +7. **Creates the message.** It freshly reads the newest message, places the + delivery after it by timestamp, carries the previous thread id, and creates + the assistant message. A 409 counts as success. +8. **Notifies.** Delivered messages use the existing chat-response notification + helper. Undeliverable and expired runs use the new + `workflow_chat_delivery` notification type. Keys make each notice exactly + once. +9. **Finishes.** It closes the record as `delivered`, `undeliverable` or + `expired`, and reconciles with the live control in that final write so a + resume that landed mid-delivery can reopen the next generation. + +Attempts are capped at 8. What happens at the cap depends on whether the worker +managed to claim the record: + +- **Before a claim**, when the runtime control can't be read, the worker closes + the record silently as `undeliverable` with `delivery_failed`. If an earlier + attempt already created the message, the record closes as `delivered` without + a notice. No outcome was read, so there is nothing to tell the user. +- **After a claim**, the worker closes with one best-effort notice and never + posts a second message. If the message was created, the record closes as + `delivered` with the chat notice. An expiry closes as `expired` with the + expired notice. Otherwise the record closes as `undeliverable` with + `delivery_failed` and the undeliverable notice. + +### Compose, fencing and token logging + +For `kind: result`, the worker reads the result with `include_excerpts=True` and +an initial 48 KB excerpt budget. If the prompt does not fit a model's window, it +steps down through the same 6a budget ladder: 24 KB, 12 KB and 6 KB. If no +budget fits, it posts the fixed status note instead of result text. + +The question is the original user message if it still exists and was not +retracted. Otherwise the worker uses the fixed fallback: + +```text +Summarize what this workflow run produced. +``` + +No chat history is sent. The dropped plan-step-goal fallback is not in the +implementation. + +The model chain is: + +1. the request's stored model selection, re-authorized for the requester with a + headless execution identity for the conversation; +2. the default model, skipped when it resolves to the same binding; +3. a trimmed fallback if no model answers: the fixed sentence + `I couldn't summarize the result, so here's the start of it:`, then the first + 2,000 characters of the excerpt in a code block whose fence is longer than + any run of backticks in the data. + +The worker calls 6a's `build_workflow_result_messages` with a fresh fence nonce +on every attempt. The result is fenced as untrusted data. It then calls the +model with `temperature=0.3` and `max_tokens=1500`, clips the reply at 12,000 +characters, and appends 6a's disclosure line. + +The final composed or trimmed text is content-checked as `chat_output`. A +blocked result posts the fixed `content_blocked` note instead, records a +best-effort content incident without excerpt text, and does not retract the +message. Fixed notes are application-written and are not model-checked. + +When a model returns usage, the worker logs token usage like 6a Follow up: +`token_type='chat'`, `workspace_type='personal'`, `additional_context` of +`{'workflow_result_delivery': True}`, and idempotency key +`workflow_result_delivery:{message_id}:{attempt}`. It does not log a chat +activity user message. Token logging failures never fail delivery. + +### Labels and application-owned texts + +The label line is: + +```text +Results from `{workflow name}` ยท you asked on {formatted requested_at} +``` + +Workflow names are cleaned to one line, capped at 80 characters and rendered as +inline code with backticks replaced by apostrophes. If no name remains, the +label uses `` `Workflow` ``. The timestamp uses the same formatter as 6a and the +record's time zone. + +Fixed note bodies from `delivery_note_text` are: + +| Kind | Exact text pattern | +| --- | --- | +| `failed` | `` `{name}` failed: {closed reason text}. `` | +| `cancelled` | `` `{name}` was cancelled. `` | +| `status` | `` `{name}` finished. Open the run to see its results. `` | +| `content_blocked` | `` `{name}` finished, but its results can't be shown here. Open the run to see them. `` | +| `analysis` | `` `{name}` finished with a saved analysis. Ask a follow-up question about it here. `` | +| `skipped` | `` `{name}` didn't run: no new or changed files were detected. `` | + +The failed reason comes only from a closed map: `failed`, `invalid`, +`incomplete`, `skipped`, `deadline_exceeded`, `execution_budget_exceeded`, +`repeat_iteration_limit`, `m365_authorization` and `authorization`. Unknown +failure codes use `the run stopped before it finished`. + +Notice titles and messages are fixed: + +| Case | Type | Title or preview | Message | Key | +| --- | --- | --- | --- | --- | +| Delivered | `chat_response_complete` through `create_chat_response_notification` | `Results from "X" are in your chat` as a non-empty preview | Existing helper message, opening the chat | `workflow-chat-delivery:{run_id}:{generation}` | +| Undeliverable completed/partial/status/content-blocked/analysis | `workflow_chat_delivery` | `Results from "X" are ready` | `The chat that started this run can't show it anymore. Open the run in Workflows to see the details.` | `workflow-chat-delivery-notice:{run_id}:{generation}` | +| Undeliverable failed | `workflow_chat_delivery` | `"X" didn't finish` | Same undeliverable message | Same undeliverable key | +| Undeliverable cancelled | `workflow_chat_delivery` | `"X" was cancelled` | Same undeliverable message | Same undeliverable key | +| Undeliverable skipped | `workflow_chat_delivery` | `"X" didn't run` | Same undeliverable message | Same undeliverable key | +| Expired | `workflow_chat_delivery` | `"X" didn't finish in time to post to chat` | `The run didn't finish in time to post to the chat. Open it in Workflows to see where it stands.` | `workflow-chat-delivery-notice:{run_id}:expired` | + +Undeliverable and expired notices link to +`/workflow-activity?workflowId=...&runId=...&scope=personal` and carry metadata +`{workflow_id, run_id, workflow_scope: 'personal', delivery_status}`. Until 6b-2 +adds a V2 case, the V2 bell labels the new type generically as `Notification`. + +### Message shape and placement + +The delivered message document is an assistant message shaped like a normal chat +answer: + +```json +{ + "id": "assistant_workflow_delivery_...", + "conversation_id": "...", + "role": "assistant", + "content": "Results from `Weekly digest` ...", + "timestamp": "2026-01-05T02:37:11.000001", + "model_deployment_name": "gpt-4o", + "augmented": false, + "hybrid_citations": [], + "hybridsearch_query": null, + "agent_citations": [], + "web_search_citations": [], + "user_message": null, + "metadata": { + "workflow_delivery": {"version": 1}, + "token_usage": {}, + "user_info": {"user_id": "..."}, + "thread_info": {"thread_id": "...", "previous_thread_id": "...", "active_thread": true, "thread_attempt": 1} + } +} +``` + +The worker reads the newest message with `ORDER BY c.timestamp DESC`; a failed +read defers rather than treating the chat as empty. The delivery timestamp is +later than the newest timestamp by one microsecond when necessary, so loaders +sort it last. The message does not carry `metadata.orchestration`; it is not an +orchestration answer. + +### Retry and Edit refusal + +`is_workflow_delivery_message(message)` is true when the id starts with +`assistant_workflow_delivery_` or `metadata.workflow_delivery` is present. +Classic Retry and Edit refuse those messages before replay work begins: + +| Operation | Response | +| --- | --- | +| Retry | HTTP 400 `{"error": "A workflow run posted this message, so it can't be retried here. To run the workflow again, open the run in Workflows.", "code": "workflow_delivery_retry_unsupported"}` | +| Edit | HTTP 400 `{"error": "A workflow run posted this message, so it can't be edited. Ask a new question instead.", "code": "workflow_delivery_edit_unsupported"}` | + +V2 orchestration retry/edit routes take an orchestration run id, not a message +id, so a delivery message id is not a delivery trigger there and returns the +existing 404 path. + +### Masking and lineage + +A delivered result answer and an analysis note carry the same lineage keys as a +6a Follow up answer: + +- `metadata.workflow_result`: the reader's public descriptor snapshot. +- `metadata.workflow_result_contexts`: an array containing the context built + from that descriptor. + +Those keys make later chat turns inherit the result lineage and make shared +sanitizers withhold the delivered content when the run is gone, the result +changed, source access is lost, storage cannot confirm access, or the chat is no +longer private. Status, failed, cancelled, skipped and content-blocked notes +carry no result text and no 6a lineage keys. + +Every delivered message carries `metadata.workflow_delivery`, described in the +message metadata contract below. The id prefix is a second marker and survives +content-review retraction. + +### Expiry, deferral, sweep and lease + +A record can become `expired` when the runtime pauses with `deadline_exceeded`, +or when `now > expires_at` and the run is still non-terminal. The worker posts +nothing and sends one expired notice. If a run finished but stayed undelivered +past `expires_at`, it becomes `undeliverable` with +`expired_before_delivery` and a ready notice. Expired records never reopen. + +The worker delays delivery while the chat may still be receiving the +orchestration answer or a user reply: shared stream activity is active and fresh +within 90 seconds, the newest message is a user message less than 10 minutes +old, or the newest-message read fails. The first uncounted deferral records +`first_deferred_at` and retries after 30 seconds. After 15 minutes, soft +activity signals are ignored and delivery continues; read failures still defer +and count as attempts. + +`signal_workflow_chat_delivery(user_id, run_id)` puts a bounded in-process hint +onto a 256-entry queue and wakes the delivery loop. Hints are point reads plus +an ETag claim, so they do not need the distributed sweep lock. + +`run_workflow_chat_delivery_loop(app)` runs inside the Flask app context when an +app is provided. On each wake it processes hints and, every 30 seconds, attempts +to acquire the distributed lock `workflow_chat_delivery_sweep` for 120 seconds. +With the lock it runs one cross-partition sweep over due `chat_delivery` +records. The sweep uses `TOP 25`, processes at most 8 runs or 90 seconds, and +uses a 120-second `_ts` grace so normal runtime projection can win. A claim +lease lasts 300 seconds. A slow model call may be duplicated after lease expiry, +but only the worker that still owns the lease can persist the message phase. + +### Terminal-write coverage + +| # | Terminal or reopen write | Coverage | +| --- | --- | --- | +| 1 | Runner cancellation saves and cancellation merge writes | Guarded save preserves delivery; runtime projection reconciles and signals; sweep catches pending terminal rows after grace. | +| 2 | Runner failure saves (`failed`) | Guarded save, projection trigger and sweep backstop. | +| 3 | Runner success and partial success saves (`completed`, `completed_partial`) | Guarded save, projection to `ready` with `kind: result`, signal and sweep. | +| 4 | Runner final exception save (`failed`) | Guarded save, projection and sweep. | +| 5 | Non-terminal runner progress saves | Guarded save preserves `chat_delivery` and `chat_invocation`, including the double-fault path. | +| 6 | Durable projection `_project_runtime_run` | Keeps stored delivery, reconciles under ETag and signals when a generation becomes ready. | +| 7 | Runtime deadline expiry on the control | Next continue projects `paused` + `deadline_exceeded`; sweep also catches `expires_at`. | +| 8 | Durable cancel and reject decisions | Projection maps the terminal state to cancelled or failed note kinds and signals. | +| 9 | Scheduler continuations | Covered through durable projection. | +| 10 | Personal cancel routes | Save through the guarded path; sweep catches pending terminal rows. | +| 11 | `runtime/resume` | Control version increases; projection reopens delivered or undeliverable records when `version > generation` and not deleted. | +| 12 | `runtime/decision` approve/reject | Approve returns to running; reject terminal states are handled through projection. | +| 13 | `resume-failed` | Not a trigger. It returns 409 for durable runs, and any non-durable replacement would have no `chat_invocation`. | +| 14 | Workflow deletion | Deleted workflow with a run present closes silently as `workflow_deleted`; deleted runs have nothing to deliver. | +| 15 | MCP and group workflow saves | Out of scope because they do not carry `chat_delivery`; group runs are for a later phase. | +| 16 | Signal failure | Projection never fails because the signal is non-raising; the sweep catches up later. | + +## Status route contract for 6b-2 + +### Route, decorators and query parameters + +Path: + +```text +GET /api/v2/orchestration/workflow-runs/status[?conversation_id=] +``` + +Decorators, in order: + +1. `@bp.route(..., methods=["GET"])` +2. `@swagger_route(security=get_auth_security())` +3. `@login_required` +4. `@user_required` +5. `@enabled_required('allow_user_workflows')` +6. `@workflow_user_required` + +The optional `conversation_id` may appear once. It is trimmed and must match +`[A-Za-z0-9_-]{1,128}`. Missing means global mode. Empty, repeated, too long or +malformed values return HTTP 400: + +```json +{"error": "The conversation ID is not valid.", "code": "invalid_conversation_id"} +``` + +An invalid server-side user id or storage/settings failure returns HTTP 503: + +```json +{"error": "Workflow run status isn't available right now. Try again later.", "code": "workflow_run_status_unavailable"} +``` + +### Query modes and limits + +All reads are owner-only by construction: the query uses +`partition_key=`, filters `c.user_id = @user_id`, filters +`c.trigger_source = 'chat_orchestration'`, and requires +`IS_DEFINED(c.chat_invocation)`. It projects the needed fields only; it does not +use `SELECT *`. + +| Mode | Query limit | Returned rows | Filter | Order | +| --- | --- | --- | --- | --- | +| With `conversation_id` | `TOP 21` | Up to 20 | That chat's started runs | `ORDER BY c.chat_invocation.requested_at DESC` | +| Without `conversation_id` | `TOP 51` | Up to 50 | Non-terminal runs, open delivery records, or deliveries from the last 10 minutes | Same order | + +`truncated` is true when the query returned more than the public limit. The +runs container uses the default indexing policy; every chat invocation writer +sets `requested_at`. + +For in-flight rows, the route may do at most 10 live runtime reads per request. +It may do at most 20 workflow document reads per request, cached per distinct +workflow and counted separately. When a live read fails or the cap is reached, +the row falls back to stored projection values with `live: false` and fixed +closed codes. + +`available` reflects the Phase 5 `workflow_run_settings_gate` at request time. +Rows are still returned when `available` is `false`, so an in-flight card does +not vanish when an administrator changes the setting mid-run. + +### Response example + +```json +{ + "available": true, + "runs": [ + { + "workflow_id": "wf_123", + "workflow_scope": "personal", + "run_id": "run_123", + "workflow_name": "Weekly digest", + "conversation_id": "conv_123", + "orchestration_run_id": "orch_123", + "step_id": "step_2", + "requested_at": "2026-01-05T02:30:00Z", + "status": "running", + "phase": "running", + "runtime_version": 3, + "step_index": 2, + "step_count": 5, + "step_label": null, + "started_at": "2026-01-05T02:30:15Z", + "completed_at": null, + "elapsed_seconds": 416, + "waiting": null, + "delivery": { + "status": "pending", + "generation": 3, + "message_id": null, + "delivered_at": null, + "reason": null + }, + "error": null, + "error_code": null, + "retry_blocked": null, + "actions": { + "cancel": true, + "retry": false, + "approve": false, + "open_run": true + }, + "live": true + } + ], + "checked_at": "2026-01-05T02:37:11Z", + "truncated": false +} +``` + +### Field rules + +| Field | Rule | +| --- | --- | +| `workflow_id`, `run_id`, `conversation_id`, `orchestration_run_id`, `step_id` | Bounded identifiers from the projected run row. Rows missing required ids are omitted. | +| `workflow_scope` | Always `personal`. | +| `workflow_name` | Cleaned to one line and capped at 80 characters, or `Workflow`. | +| `requested_at`, `started_at`, `completed_at`, `checked_at` | Seconds-precision UTC strings when known. `completed_at` is null for non-terminal rows. | +| `runtime_version` | Live runtime version when a live read succeeded, otherwise the run's `runtime_version`, otherwise the stored projection version. This is the `expected_version` for resume. | +| `step_index`, `step_count` | From live progress, stored runtime progress, or stored run progress. | +| `step_label` | Always `null` in this implementation. | +| `elapsed_seconds` | From `started_at` to `completed_at` for terminal rows, or to `checked_at` for active rows. Null when `started_at` is missing. | +| `live` | True only when a live runtime read succeeded for an in-flight row. | + +Status mapping: + +| Runtime state | `status` | `phase` | +| --- | --- | --- | +| `queued` | `queued` | `running` | +| `running`, `cancelling`, `waiting_recovery` and unknown states | `running` | `running` | +| Other waiting states | `waiting` | `needs_you` | +| `completed` | `completed` | `finished` | +| `completed_partial` | `completed_partial` | `finished` | +| `failed`, `invalid`, `incomplete`, `skipped` | `failed` | `failed` | +| `cancelled` or `canceled` | `cancelled` | `cancelled` | +| Non-terminal row with delivery `expired`, or paused with `deadline_exceeded` | `expired` | `failed` | + +Waiting mapping: + +| Runtime state | `waiting.reason` | `waiting.action` | `waiting.gate_id` | +| --- | --- | --- | --- | +| `waiting_approval` | `approval` | `approve` | Runtime gate id or null | +| `awaiting_sign_in` | `microsoft_365_reconnect` | `reconnect` | Runtime gate id or null | +| Other Microsoft 365 waiting states | `microsoft_365_approval` | `open_run` | Runtime gate id or null | +| `waiting_output` | `output_review` | `open_run` | Runtime gate id or null | +| `waiting_recovery` | `recovery` | `open_run` | Runtime gate id or null | +| `paused` with `deadline_exceeded` | `deadline_exceeded` | `open_run` | Runtime gate id or null | +| Other `paused` | `paused` | `open_run` | Runtime gate id or null | + +`waiting` is null for every other state. It is not tied to `phase`: a +`waiting_recovery` row reports `status: running` and `phase: running`, because +the runtime recovers it without the user, but still carries +`waiting: {reason: recovery, action: open_run}`. 6b-2 should show the waiting +reason without treating the row as needing the user. + +Delivery mapping: + +| Stored delivery state | Row `delivery.status` | Other fields | +| --- | --- | --- | +| Missing or invalid | `not_applicable` | Null generation, message, delivered time and reason. | +| `ready` | `pending` | The generation is included when known. | +| `pending`, `delivering`, `undeliverable`, `expired` | Same value | `reason` is returned only when it is one of the closed delivery reason codes. | +| `delivered` | `delivered` | `message_id` is returned only when it starts with `assistant_workflow_delivery_`; `delivered_at` is seconds precision. | + +`error` and `error_code` are fixed and never come from the run's raw error text +or exception text. `expired` returns `It reached its time limit.` with +`deadline_exceeded`. Failed rows use the same closed failure map as notes. + +`retry_blocked` is null or one of these codes, checked in this order: +`retry_unavailable`, `workflow_deleted`, `not_resumable`, +`deadline_exceeded`, `workflow_definition_changed` or +`workflow_already_running`. The implementation also returns +`retry_unavailable` for failed workflow reads, workflow-read cap exhaustion, +missing definition revisions, or revision-computation failures. `skipped` and +`cancelled` are never retryable. + +`actions` mapping: + +| Action | True when | +| --- | --- | +| `cancel` | The row is non-terminal, not expired and not `cancelling`. | +| `retry` | State is `failed`, `invalid` or `incomplete`, and `retry_blocked` is null. | +| `approve` | Waiting action is `approve`, a `gate_id` is present, and the row is not expired. | +| `open_run` | Always true. | + +6b-2 must treat the row as a snapshot. Retry buttons should still handle a 409 +from resume, including `workflow_definition_changed`, +`workflow_already_running`, `workflow_deleting`, `stale_version`, +`invalid_state`, `deadline_exceeded` and `request_conflict`, plus 400, 403, 404 +and 503 responses. + +### Personal run routes used by 6b-2 + +6b-2 calls existing personal durable run routes: + +- `GET /api/user/workflows//runs//runtime` +- `POST /api/user/workflows//runs//runtime/decision` +- `POST /api/user/workflows//runs//runtime/resume` with + body `{expected_version, request_id}`, where `request_id` is a UUID generated + by the client for that resume request. +- `POST /api/user/workflows//runs//cancel` + +The older `POST /api/user/workflows//runs//resume-failed` +route is not used for delivery. It returns 409 for durable runs and would create +a new run id for non-durable runs. + +## Message metadata contract + +### `metadata.workflow_delivery` + +Every posted delivery message, including fixed notes, carries: + +```json +{ + "version": 1, + "kind": "result", + "workflow_id": "wf_123", + "workflow_scope": "personal", + "run_id": "run_123", + "generation": 3, + "run_status": "completed", + "orchestration_run_id": "orch_123", + "step_id": "step_2", + "requested_at": "2026-01-05T02:30:00Z" +} +``` + +This metadata is a snapshot. 6b-2 uses it to identify delivery messages, hide +plain Retry and Edit, render **Open run**, and join the live status row by +`run_id`, `orchestration_run_id` and `step_id`. The Retry/Edit refusal checks +this key, and also checks the `assistant_workflow_delivery_` id prefix. + +### Delivered result answer + +A delivered composed or trimmed result has: + +- `metadata.workflow_delivery.kind: "result"`; +- `metadata.workflow_result`: the 6a public descriptor snapshot; +- `metadata.workflow_result_contexts`: contexts used by Follow up and masking; +- `metadata.token_usage`: usage values when the model returned them; +- `metadata.user_info.user_id`; +- `metadata.thread_info` with deterministic thread id and previous thread id; +- optional content-check metadata attached by the existing chat content checker. + +6b-2 reads `metadata.workflow_result` for Follow up and reads the live status +row's `actions.retry` to decide whether to show Retry on a delivery note or +card. The metadata alone never makes Retry available. + +### Fixed note kinds + +| Note kind | 6a lineage keys | Why | +| --- | --- | --- | +| `analysis` | `workflow_result` and `workflow_result_contexts` | Saved-analysis results have no excerpts, but Follow up and masking should work from the descriptor. | +| `failed` | None | No result text is available. | +| `cancelled` | None | No result text is available. | +| `skipped` | None | No result text is posted. | +| `status` | None | The result cannot be shown here or the gate is off. | +| `content_blocked` | None | The model or trimmed result was blocked, so no result text is posted. | + +The snapshot rule applies to every kind. If the workflow is later edited, +deleted, resumed or blocked by another active run, 6b-2 must use the status row +and route responses as the live truth. + +## Configuration + +| Setting or constant | Value or behavior | +| --- | --- | +| `enable_chat_workflow_results` (**Use Workflow Results In Chat**) | Off by default. Checked before seeding delivery and again before reading result text. Uses `is_chat_workflow_results_enabled_for_user(settings, user_roles)`. | +| `allow_user_workflows` | Required by route decorators, worker access checks and the 6a gate. | +| `require_member_of_workflow_user` | Enforced through roles captured at start; role revocations after start are not visible to the headless worker. | +| `enable_chat_orchestration_workflow_runs` | Phase 5 gate that allows a plan to start a saved workflow. | +| `CHAT_DELIVERY_VERSION` | `1` | +| `DELIVERY_MESSAGE_ID_PREFIX` | `assistant_workflow_delivery_` | +| `NOTIFICATION_TYPE` | `workflow_chat_delivery` | +| `WORKFLOW_SCOPE` | `personal` | +| `DELIVERY_GRACE_SECONDS` / `DEFAULT_DEADLINE_SECONDS` / `MAX_DEADLINE_SECONDS` | `86400` / `86400` / `86400` | +| `LEASE_SECONDS` / `SWEEP_INTERVAL_SECONDS` / `SWEEP_LOCK_SECONDS` | `300` / `30` / `120` | +| `SWEEP_TOP` / `SWEEP_MAX_RUNS` / `SWEEP_MAX_SECONDS` / `SWEEP_TS_GRACE_SECONDS` | `25` / `8` / `90` / `120` | +| `MAX_ATTEMPTS` / `BACKOFF_SECONDS` | `8` / `(30, 60, 120, 240, 480, 900, 900, 900)` | +| `DEFER_RETRY_SECONDS` / `DEFER_MAX_SECONDS` | `30` / `900` | +| `STREAM_META_FRESH_SECONDS` / `RECENT_USER_MESSAGE_SECONDS` | `90` / `600` | +| `HINT_QUEUE_MAX` / `HISTORY_MAX` | `256` / `5` | +| `COMPOSE_TEMPERATURE` / `COMPOSE_MAX_TOKENS` / `REPLY_MAX_CHARS` | `0.3` / `1500` / `12000` | +| `TRIMMED_FALLBACK_CHARS` / `EXCERPT_BUDGET_BYTES` | `2000` / `48 * 1024` | +| Status-route `CONVERSATION_LIMIT` / `GLOBAL_LIMIT` | `20` / `50` | +| Status-route `LIVE_READ_LIMIT` / `WORKFLOW_READ_LIMIT` | `10` / `20` | +| Status-route `RECENT_DELIVERY_SECONDS` | `600` | + +## File structure + +| File | Purpose | +| --- | --- | +| `application\single_app\functions_workflow_chat_delivery.py` | Pure delivery contract: constants, text builders, metadata, seed, merge, reconcile, message ids, notification keys and hint queue. | +| `application\single_app\functions_workflow_chat_delivery_worker.py` | Delivery worker: claim, re-checks, deferral, compose, content safety, unread mark, message create, notices, sweep and outcomes. | +| `application\single_app\functions_workflow_chat_delivery_status.py` | Owner-only status route payload, queries, live-read caps and row shaping for 6b-2. | +| `application\single_app\functions_orchestration_workflow_runs.py` | Seeds delivery at Phase 5 start and changes the started-run note only when every started run will post back. | +| `application\single_app\functions_workflow_runtime.py` | Accepts `chat_delivery` in the durable queue, finalizes it into the first run write, reconciles projections and signals delivery. | +| `application\single_app\functions_personal_workflows.py` | Guarded save for chat-started runs that preserves stored delivery records. | +| `application\single_app\functions_notifications.py` | Adds `workflow_chat_delivery` and idempotency pass-through for chat response notices. | +| `application\single_app\functions_conversation_unread.py` | Adds `mark_conversation_unread_guarded`. | +| `application\single_app\route_backend_conversations.py` | Refuses Retry and Edit on delivery messages. | +| `application\single_app\route_backend_orchestration.py` | Adds the V2 status route. | +| `application\single_app\background_tasks.py` | Registers the delivery loop in the background task host and scheduler host. | +| `application\single_app\config.py` | Tracks version `0.261.227`. | + +## Usage + +### Enable or configure + +1. Turn on **Enable Personal Workflows**. If **Require WorkflowUser App Role** is + on, assign the `WorkflowUser` role to the people who should use this. +2. Turn on **Run Workflows From Chat** under **Admin Settings > Orchestration** + so chat plans can start saved workflows. +3. Turn on **Use Workflow Results In Chat** under **Admin Settings > Workflow** + so chat-started workflow results can be posted back. +4. Make sure the workflow has **Durable execution** on. Non-durable runs are not + started by Phase 5 and are not delivery candidates. + +### What users see + +A user asks in a private chat, "Run my weekly digest now," with orchestration on. +After approval, the answer lists the workflow as started and says: + +```text +I'll post the results here when the run finishes. You can also follow it in the workflow's run history in Workflows. +``` + +Later, when the run completes, the server posts a new assistant message at the +end of that chat, with a label such as: + +```text +Results from `Weekly digest` ยท you asked on Sun Jan 4, 2026, 9:30 PM EST +``` + +The chat is marked unread and the user receives one bell notification. If the +chat was deleted, shared or no longer belongs to the requester, no message is +posted; the user receives one workflow notice that links to the run. If the +workflow was deleted or the runtime control is missing, the record closes +silently because the run link would be dead or unknowable. + +## Testing and validation + +When this documentation was written, these workflow chat delivery tests existed +under `functional_tests\`: + +- `test_workflow_chat_delivery_concurrency.py` +- `test_workflow_chat_delivery_contract.py` +- `test_workflow_chat_delivery_control_pins.py` +- `test_workflow_chat_delivery_imports.py` +- `test_workflow_chat_delivery_loop.py` +- `test_workflow_chat_delivery_notice_and_unread.py` +- `test_workflow_chat_delivery_off_golden.py` +- `test_workflow_chat_delivery_placement_and_masking.py` +- `test_workflow_chat_delivery_projection_and_seed.py` +- `test_workflow_chat_delivery_refusals.py` +- `test_workflow_chat_delivery_save_guard.py` +- `test_workflow_chat_delivery_status_route.py` +- `test_workflow_chat_delivery_worker.py` + +Shared fakes live in `functional_tests\test_support\workflow_chat_delivery_fakes.py`. + +Route-policy changes are covered by: + +- `functional_tests\route_tests\test_route_blueprint_policy_inventory.py` +- `functional_tests\route_tests\test_route_unauthenticated_policy_contract.py` +- `functional_tests\route_tests\test_route_policy_test_coverage.py` + +The docs inventory was regenerated with `scripts\build_docs_inventory.py`, and +the documentation coverage and site-quality tests were run. Mutation testing +results for the delivery implementation are recorded in the pull request. + +## Known limitations and residual risks + +- A third consecutive storage read failure in the double-fault guarded-save path + can still wipe `chat_delivery`. In that case no message is posted, and the + status route reports the run as `not_applicable`. +- Mark-before-create has small windows: the chat may look unread just before the + message exists, a permanent create failure can leave a dangling unread mark, + and a chat deleted between mark and create can leave an orphaned message. +- A crash after unread is marked but before the message is created can cause a + second compose call and possibly a second content-safety incident. The first + composed text is not persisted. +- Version-1 runtime controls do not always have `deadline_at`. Their expiry is + computed from the admitted deadline; a run that finishes after `expires_at` + gets a ready notice rather than a chat post. +- Role revocations after start are not seen by the headless worker, because it + has no live browser session. It uses the roles captured at start, as durable + background execution already does. +- Owner-visible run history can show the raw `chat_delivery` record. That record + contains roles, model ids, lease ids and codes, but never result text. +- The workflow's own alert rules still fire for the same run, so a user may get + both a workflow alert and a chat post. +- Group workflow delivery, in-plan waiting for run completion, and a reusable + generic unread helper beyond this scenario are out of scope for this phase. + +## Deviations from the plan + +The implementation differs from the approved plan in these documented ways: + +- The delivered bell preview is fixed non-empty text (`Results from "X" are in your chat`), never result text. +- The worker marks the chat unread before creating the message, and keeps + `skip_if_read_since` for retries after a read. +- The plan-step-goal question fallback was dropped; the worker uses the original + user message or the fixed generic instruction. +- Prompt size is pre-checked before compose, and the excerpt budget steps down + from 48 KB to 24 KB, 12 KB and 6 KB. +- Attempt exhaustion before a claim can complete closes silently at 8 attempts. +- Notices use deterministic idempotency keys. An excerpt that still cannot fit + becomes a status note. +- The note kind is derived from the terminal run state. Expired records never + reopen. +- The delivery loop runs inside the Flask app context. Workflow reads in the + worker are direct container reads. +- Retry/Edit refusal payloads are `{error, code}` with HTTP 400. +- Status-route errors are `{error, code}`. `cancelling` and `waiting_recovery` + report as running. Unknown runtime states fall back to stored status and then + to `running` / `running`. +- `step_label` is always `null` in the status route. The workflow-read cap is + 20, separate from the 10 live runtime reads. +- `expired` applies only to non-terminal status rows. Status-route timestamps + have seconds precision. +- The definition revision for retry eligibility is computed from the editor form + exactly as durable resume does. Approve requires a `gate_id`. +- An invalid user id returns 503. `available: false` still returns stored rows. + An empty `conversation_id=` returns 400. +- Rebuilt or linked runs keep the older Phase 5 follow-up note text. +- The workflow notice link is the classic + `/workflow-activity?workflowId=...&runId=...&scope=personal` path. +- The V2 bell shows the generic `Notification` label for + `workflow_chat_delivery` until 6b-2 adds a specific case. + +## Related + +- [Chat orchestration workflow runs](CHAT_ORCHESTRATION_WORKFLOW_RUNS.md) +- [Workflow results in chat](CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md) +- [Workflow settings](../../admin/workflow.md) +- [Orchestration settings](../../admin/orchestration.md) +- [Trigger a workflow](../../guides/trigger-a-workflow.md) +- [Ask about workflow results](../../guides/ask-about-workflow-results.md) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 9a2bae3be..47cdfa50c 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,17 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.227)** + +#### New Features + +* **Workflow results posted back to chat** + * When a chat orchestration plan starts one of the user's saved durable personal workflows, the server can now post the run's terminal outcome back into the same private chat after the run finishes, even hours later. + * The behavior is gated by **Use Workflow Results In Chat** (`enable_chat_workflow_results`), checked when the run starts and again when delivery happens, in addition to the existing Run Workflows From Chat gates. + * Delivery is exactly once per durable runtime generation: one deterministic chat message, one unread mark, and one bell notification. If the chat can no longer receive the post, the server posts nothing and sends one workflow notification with a run link instead. + * A new owner-only status route, `GET /api/v2/orchestration/workflow-runs/status`, gives 6b-2 the batched run-card contract. Retry/Edit on delivery messages are refused with fixed 400 codes. + * (Ref: #1546, #1543, `functions_workflow_chat_delivery.py`, `functions_workflow_chat_delivery_worker.py`, `functions_workflow_chat_delivery_status.py`, `functions_orchestration_workflow_runs.py`, `functions_workflow_runtime.py`, `route_backend_orchestration.py`, `route_backend_conversations.py`, [Workflow result delivery to chat](features/CHAT_WORKFLOW_RESULT_DELIVERY.md)) + ### **(v0.261.226)** #### Bug Fixes diff --git a/docs/guides/trigger-a-workflow.md b/docs/guides/trigger-a-workflow.md index 956c96be5..646400ec0 100644 --- a/docs/guides/trigger-a-workflow.md +++ b/docs/guides/trigger-a-workflow.md @@ -83,6 +83,12 @@ alerts. Stopping the plan doesn't stop a workflow it already started, so cancel from its run history if you need to. Retrying a failed step never starts a workflow twice: when the plan already started it, the retry links that run instead. +When your administrator also turns on **Use Workflow Results In Chat**, a +durable personal run started from chat can post its outcome back into that same +private chat after it finishes. The chat is marked unread, and a bell +notification opens the chat. If the chat was deleted or shared before delivery, +you get a workflow notification that opens the run instead. + A workflow that's already running isn't started again; the answer says so, and you can start it once that run finishes. If a run the plan started waits for a Microsoft 365 approval or sign-in, its status shows **Waiting**; follow diff --git a/functional_tests/route_tests/test_route_blueprint_policy_inventory.py b/functional_tests/route_tests/test_route_blueprint_policy_inventory.py index 687d326fa..683c02a61 100644 --- a/functional_tests/route_tests/test_route_blueprint_policy_inventory.py +++ b/functional_tests/route_tests/test_route_blueprint_policy_inventory.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for route blueprint policy inventory. -Version: 0.261.214 +Version: 0.261.227 Implemented in: 0.242.069 Plan editor policy coverage: 0.261.102 Selected-group context policy coverage: 0.261.126 @@ -12,6 +12,7 @@ Workflow proposal policy coverage: 0.261.207 Workflow assistant policy coverage: 0.261.208 Workflow result context policy coverage: 0.261.214 +Workflow run status policy coverage: 0.261.227 This test ensures every SimpleChat route is assigned to a Blueprint-based security policy or an explicit reviewed route exemption. @@ -537,6 +538,29 @@ def test_workflow_run_link_route_keeps_the_orchestration_security_policy() -> No assert {"swagger_route", "login_required", "user_required"} <= set(route.decorator_names) +def test_workflow_run_status_route_keeps_the_personal_workflow_security_policy() -> None: + """The run status rows stay owner-only, behind the same gates as the personal runtime routes. + + Explicit raises keep this check under ``python -O``. + """ + path = "/api/v2/orchestration/workflow-runs/status" + routes = [ + route for route in iter_route_functions() + if route.file_name == "route_backend_orchestration.py" and route.path == path + ] + if len(routes) != 1: + raise AssertionError(f"Expected one status route, found {len(routes)}.") + route = routes[0] + if route.function_name != "orchestration_workflow_run_status" or route.route_target != "bp": + raise AssertionError(f"Unexpected status route registration: {route.function_name} on {route.route_target}.") + expected = ( + "bp.route", "swagger_route", "login_required", "user_required", + "enabled_required", "workflow_user_required", + ) + if route.decorator_names != expected: + raise AssertionError(f"Status route decorators changed: {route.decorator_names}.") + + def test_content_screening_routes_keep_authenticated_blueprint_guards() -> None: """Evidence and decisions never become public, even while enrollment is off.""" routes = [ @@ -571,6 +595,7 @@ def test_content_screening_routes_keep_authenticated_blueprint_guards() -> None: test_plan_editor_routes_keep_the_orchestration_security_policy, test_workflow_proposal_routes_keep_the_orchestration_security_policy, test_workflow_run_link_route_keeps_the_orchestration_security_policy, + test_workflow_run_status_route_keeps_the_personal_workflow_security_policy, test_content_screening_routes_keep_authenticated_blueprint_guards, ] results = [] diff --git a/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py b/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py index 05f8bdfe1..bac3f520e 100644 --- a/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py +++ b/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py @@ -2,9 +2,10 @@ # test_route_unauthenticated_policy_contract.py """ Functional test for route unauthenticated access policy contract. -Version: 0.261.214 +Version: 0.261.227 Implemented in: 0.242.069 Workflow result context coverage: 0.261.214 +Workflow run status coverage: 0.261.227 This test ensures every SimpleChat route has an explicit expected unauthenticated access behavior: public, browser-session authenticated, admin-only, or external @@ -276,6 +277,25 @@ def test_workflow_result_context_requires_a_signed_in_user_session() -> None: assert {"login_required", "user_required"} <= set(matches[0].decorator_names) +def test_workflow_run_status_requires_a_signed_in_user_session() -> None: + """The run card's status rows are never public or bearer-only. + + Explicit raises keep this check under ``python -O``. + """ + path = "/api/v2/orchestration/workflow-runs/status" + matches = [route for route in iter_route_functions() if route.path == path] + + if [route.function_name for route in matches] != ["orchestration_workflow_run_status"]: + raise AssertionError(f"Unexpected status route functions: {[route.function_name for route in matches]}.") + if expected_policy(path) != "session_user_401_or_redirect": + raise AssertionError(f"Status route policy changed: {expected_policy(path)}.") + required = {"login_required", "user_required", "enabled_required", "workflow_user_required"} + if not required <= set(matches[0].decorator_names): + raise AssertionError(f"Status route is missing guards: {sorted(required - set(matches[0].decorator_names))}.") + if "accesstoken_required" in matches[0].decorator_names: + raise AssertionError("Status route must not accept bearer tokens.") + + if __name__ == "__main__": tests = [ test_every_route_has_unauthenticated_access_policy, @@ -283,6 +303,7 @@ def test_workflow_result_context_requires_a_signed_in_user_session() -> None: test_external_routes_use_bearer_auth_decorator, test_sensitive_admin_routes_have_admin_or_specialized_route_decorator, test_workflow_result_context_requires_a_signed_in_user_session, + test_workflow_run_status_requires_a_signed_in_user_session, ] results = [] for test in tests: diff --git a/functional_tests/test_orchestration_workflow_run_adapter.py b/functional_tests/test_orchestration_workflow_run_adapter.py index 025db5264..1c88e492d 100644 --- a/functional_tests/test_orchestration_workflow_run_adapter.py +++ b/functional_tests/test_orchestration_workflow_run_adapter.py @@ -1150,6 +1150,8 @@ def test_the_run_module_writes_only_through_the_queue_and_never_imports_flask(): "RUNTIME_TERMINAL_STATES", "queue_durable_workflow_run", "workflow_run_id_for_request", }, "functions_workflow_runtime_store": {"RuntimeUnavailable", "WorkflowRuntimeConflict"}, + # Use Workflow Results In Chat's role-aware gate, read only when a run is about to start. + "functions_settings": {"is_chat_workflow_results_enabled_for_user"}, } calls = _calls_by_function(tree) writes = {"create_item", "upsert_item", "replace_item", "delete_item", "patch_item", "execute_item_batch"} diff --git a/functional_tests/test_support/workflow_chat_delivery_fakes.py b/functional_tests/test_support/workflow_chat_delivery_fakes.py new file mode 100644 index 000000000..46e53db35 --- /dev/null +++ b/functional_tests/test_support/workflow_chat_delivery_fakes.py @@ -0,0 +1,988 @@ +# workflow_chat_delivery_fakes.py +""" +Reusable fakes for the workflow chat delivery tests. + +The fakes stand in for Cosmos containers, the durable runtime store, the result +reader, the chat model and the notification helpers. The delivery contract and the +worker under test stay real. Checks inside this module raise explicitly so they +still run under ``python -O``. +""" + +import copy +import itertools +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from azure.cosmos.exceptions import ( + CosmosAccessConditionFailedError, + CosmosHttpResponseError, + CosmosResourceExistsError, + CosmosResourceNotFoundError, +) + +from functions_workflow_chat_delivery import ( + build_chat_delivery_seed, + finalize_chat_delivery_seed, +) +from functions_conversation_unread import mark_conversation_unread_guarded +from functions_orchestration_memory import conversation_is_private +from functions_workflow_chat_delivery_worker import WorkflowChatDeliveryServices + + +USER = "user-delivery-1" +OTHER_USER = "user-delivery-2" +WORKFLOW_ID = "workflow-delivery-1" +RUN_ID = "run-delivery-1" +CONVERSATION_ID = "conversation-delivery-1" +USER_MESSAGE_ID = "message-user-1" +ORCHESTRATION_RUN_ID = "orchestration-run-1" +ATTEMPT_ROOT_RUN_ID = "orchestration-run-1" +STEP_ID = "step-1" +REQUESTED_AT = "2026-05-04T14:05:00+00:00" +NOW = "2026-05-04T15:00:00+00:00" + + +class FakeCheckFailed(AssertionError): + """A fake saw a call it was not prepared for.""" + + +def require(condition, message): + if not condition: + raise FakeCheckFailed(message) + + +_COSMOS_ERRORS = { + 404: CosmosResourceNotFoundError, + 409: CosmosResourceExistsError, + 412: CosmosAccessConditionFailedError, +} + + +def cosmos_error(status_code, message="fake cosmos error"): + error_type = _COSMOS_ERRORS.get(status_code, CosmosHttpResponseError) + error = error_type(status_code=status_code, message=message) + error.status_code = status_code + return error + + +def iso(value): + return value.astimezone(timezone.utc).isoformat() + + +def parse_iso(value): + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + + +def shift(value, **delta): + return iso(parse_iso(value) + timedelta(**delta)) + + +class FakeClock: + """A settable ISO clock; ``advance`` moves it forward.""" + + def __init__(self, start=NOW): + self.now = start + + def __call__(self): + return self.now + + def advance(self, **delta): + self.now = shift(self.now, **delta) + return self.now + + +class FakeContainer: + """An in-memory container with ETag, 404, 409 and 412 semantics.""" + + def __init__(self, name, items=None, query_handler=None, clock=None): + self.name = name + self.items = {} + self.calls = [] + self.query_calls = [] + self.query_handler = query_handler + self.clock = clock + self.fail_next = {} + self._etags = itertools.count(1) + for item in items or []: + self.put(item) + + def _key(self, item_id): + return str(item_id) + + def _stamp(self, body): + stored = copy.deepcopy(body) + stored["_etag"] = f"etag-{self.name}-{next(self._etags)}" + if self.clock is not None: + stored["_ts"] = int(parse_iso(self.clock()).timestamp()) + else: + stored.setdefault("_ts", 1777900000) + return stored + + def put(self, body): + stored = self._stamp(body) + self.items[self._key(stored["id"])] = stored + return copy.deepcopy(stored) + + def get(self, item_id): + stored = self.items.get(self._key(item_id)) + return copy.deepcopy(stored) if stored is not None else None + + def fail(self, operation, *errors): + """Queue errors for the next calls of ``operation``.""" + self.fail_next.setdefault(operation, []).extend(errors) + + def _maybe_fail(self, operation): + queued = self.fail_next.get(operation) or [] + if queued: + error = queued.pop(0) + if callable(error) and not isinstance(error, BaseException): + error = error() + if error is not None: + raise error + + def read_item(self, item=None, partition_key=None, **kwargs): + self.calls.append(("read_item", item, partition_key)) + self._maybe_fail("read_item") + stored = self.items.get(self._key(item)) + if stored is None: + raise cosmos_error(404, "not found") + return copy.deepcopy(stored) + + def replace_item(self, item=None, body=None, etag=None, match_condition=None, **kwargs): + item_id = item.get("id") if isinstance(item, dict) else item + self.calls.append(("replace_item", item_id, etag)) + self._maybe_fail("replace_item") + stored = self.items.get(self._key(item_id)) + if stored is None: + raise cosmos_error(404, "not found") + if etag is not None and stored.get("_etag") != etag: + raise cosmos_error(412, "precondition failed") + updated = self._stamp(body) + self.items[self._key(item_id)] = updated + return copy.deepcopy(updated) + + def create_item(self, body=None, **kwargs): + self.calls.append(("create_item", body.get("id"))) + self._maybe_fail("create_item") + if self._key(body.get("id")) in self.items: + raise cosmos_error(409, "conflict") + return self.put(body) + + def upsert_item(self, body=None, **kwargs): + self.calls.append(("upsert_item", body.get("id"))) + self._maybe_fail("upsert_item") + return self.put(body) + + def delete_item(self, item=None, partition_key=None, **kwargs): + item_id = item.get("id") if isinstance(item, dict) else item + self.calls.append(("delete_item", item_id)) + self._maybe_fail("delete_item") + if self.items.pop(self._key(item_id), None) is None: + raise cosmos_error(404, "not found") + + def query_items(self, query=None, parameters=None, partition_key=None, + enable_cross_partition_query=None, **kwargs): + self.query_calls.append({ + "query": query, + "parameters": copy.deepcopy(parameters or []), + "partition_key": partition_key, + "enable_cross_partition_query": enable_cross_partition_query, + }) + self._maybe_fail("query_items") + if self.query_handler is None: + raise FakeCheckFailed(f"{self.name}: unexpected query {query!r}") + params = {entry["name"]: entry["value"] for entry in parameters or []} + rows = self.query_handler(self, query, params, partition_key) + return iter([copy.deepcopy(row) for row in rows]) + + def writes(self, operation=None): + names = ("replace_item", "create_item", "upsert_item", "delete_item") + return [call for call in self.calls if call[0] in names and (operation is None or call[0] == operation)] + + +class FakeRuntimeConflict(Exception): + def __init__(self, code="conflict"): + super().__init__(code) + self.code = code + + +class FakeRuntimeUnavailable(Exception): + pass + + +class FakeResultUnavailable(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +class FakeModelError(Exception): + pass + + +class FakeRuntime: + """Runtime controls by run ID. ``factory`` returns a store whose ``read`` honors tombstones.""" + + def __init__(self): + self.controls = {} + self.errors = {} + self.reads = [] + + def set(self, run_id=RUN_ID, *, state="completed", version=5, deleted=False, gate_reason_code=None, + deadline_at=None, phase=None, schema_version=2): + control = {"state": state, "version": version, "deleted": deleted, "schema_version": schema_version} + if phase is not None: + control["phase"] = phase + if gate_reason_code is not None: + control["gate"] = {"reason_code": gate_reason_code} + if deadline_at is not None: + control["limits"] = {"deadline_at": deadline_at} + self.controls[run_id] = control + return copy.deepcopy(control) + + def update(self, run_id=RUN_ID, **changes): + control = self.controls[run_id] + control.update(changes) + return copy.deepcopy(control) + + def tombstone(self, run_id=RUN_ID): + control = self.controls[run_id] + control.update({"deleted": True, "state": "cancelled", "version": control["version"] + 1}) + return copy.deepcopy(control) + + def remove(self, run_id=RUN_ID): + self.controls.pop(run_id, None) + + def fail(self, run_id, *errors): + self.errors.setdefault(run_id, []).extend(errors) + + def factory(self, user_id, workflow_id, run_id): + runtime = self + + class _Store: + def read(self, allow_deleted=False): + runtime.reads.append((user_id, workflow_id, run_id, allow_deleted)) + queued = runtime.errors.get(run_id) or [] + if queued: + raise queued.pop(0) + control = runtime.controls.get(run_id) + if control is None or (control.get("deleted") and not allow_deleted): + raise FakeRuntimeConflict("not_found") + return copy.deepcopy(control) + + return _Store() + + +RESULT_SHA = "a" * 64 +RESULT_TEXT = "The digest found three new files and one changed report." + + +def make_result(*, text=RESULT_TEXT, budget=None, saved_inputs=None, truncated=False, partial=False, + analysis_only=False, skipped_reports=False, sha=RESULT_SHA): + descriptor = { + "workflow_id": WORKFLOW_ID, + "run_id": RUN_ID, + "result_sha256": sha, + "workflow_name": "Daily digest", + "status": "completed", + "completed_at": "2026-05-04T14:30:00+00:00", + } + return { + "descriptor": descriptor, + "excerpts": [{"text": text}] if text else [], + "truncated": truncated, + "partial": partial, + "analysis_only": analysis_only, + "skipped_reports": skipped_reports, + "saved_inputs": list(saved_inputs or []), + "budget": budget, + } + + +class FakeResultReader: + """``read_workflow_result``: queued outcomes first, then the default result at the asked budget.""" + + def __init__(self): + self.calls = [] + self.outcomes = [] + self.text = RESULT_TEXT + self.saved_inputs = None + + def __call__(self, user_id, workflow_id, run_id, *, expected_sha256=None, include_excerpts=True, + excerpt_budget_bytes=None): + self.calls.append({ + "user_id": user_id, + "workflow_id": workflow_id, + "run_id": run_id, + "expected_sha256": expected_sha256, + "include_excerpts": include_excerpts, + "budget": excerpt_budget_bytes, + }) + if self.outcomes: + outcome = self.outcomes.pop(0) + if isinstance(outcome, BaseException): + raise outcome + if callable(outcome): + return outcome(excerpt_budget_bytes) + return copy.deepcopy(outcome) + return make_result(text=self.text, budget=excerpt_budget_bytes, saved_inputs=self.saved_inputs) + + +def fake_workflow_result_context(descriptor): + if not isinstance(descriptor, dict) or not descriptor.get("result_sha256"): + raise FakeResultUnavailable("workflow_result_invalid") + return { + "workflow_id": descriptor["workflow_id"], + "run_id": descriptor["run_id"], + "result_sha256": descriptor["result_sha256"], + } + + +def fake_format_run_time(value, time_zone): + if not value: + return "" + return f"May 4, 2026 at 10:05 AM ({time_zone})" + + +class FakeDisclosure: + def __init__(self): + self.calls = [] + + def __call__(self, descriptor, time_zone, *, truncated=False, partial=False, analysis_only=False, + skipped_reports=False): + self.calls.append({ + "descriptor": copy.deepcopy(descriptor), + "time_zone": time_zone, + "truncated": truncated, + "partial": partial, + "analysis_only": analysis_only, + "skipped_reports": skipped_reports, + }) + flags = [name for name, value in (("truncated", truncated), ("partial", partial)) if value] + suffix = f" ({', '.join(flags)})" if flags else "" + return f"Source: workflow run from May 4, 2026{suffix}." + + +class FakePromptBuilder: + """``build_workflow_result_messages``: records the excerpt budget, question and nonce.""" + + def __init__(self): + self.calls = [] + self.error = None + + def __call__(self, settings, result, history, question, time_zone, *, nonce): + self.calls.append({"budget": result.get("budget"), "question": question, "nonce": nonce, "time_zone": time_zone}) + if self.error is not None: + raise self.error + excerpt = (result.get("excerpts") or [{}])[0].get("text", "") + return [ + {"role": "system", "content": f"Answer from the fenced result. Fence code: {nonce}.", "budget": result.get("budget")}, + {"role": "user", "content": f"<<{nonce}>>\n{excerpt}\n<>\n\n{question}"}, + ] + + +class FakeNonces: + def __init__(self): + self.issued = [] + + def __call__(self): + nonce = f"nonce-{len(self.issued) + 1}" + self.issued.append(nonce) + return nonce + + +class FakeBudget: + """``calculate_workflow_context_budget``: fits when the excerpt budget is at most ``max_budget``.""" + + def __init__(self): + self.calls = [] + self.max_budget = None + + def __call__(self, messages, model_name, provider=None, output_tokens=None): + budget = messages[0].get("budget") if messages else None + self.calls.append({"budget": budget, "model": model_name, "output_tokens": output_tokens}) + fits = self.max_budget is None or (budget is not None and budget <= self.max_budget) + return {"decision": "full_input" if fits else "blocked"} + + +class FakeUsage: + def __init__(self, prompt_tokens, completion_tokens, total_tokens): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + self.total_tokens = total_tokens + + +class FakeChatMessage: + def __init__(self, content, refusal=None): + self.content = content + self.refusal = refusal + + +class FakeChoice: + def __init__(self, message, finish_reason): + self.message = message + self.finish_reason = finish_reason + + +class FakeResponse: + def __init__(self, choices, usage): + self.choices = choices + self.usage = usage + + +COMPOSED_REPLY = "Your daily digest found three new files and one changed report." + + +def fake_response(text=COMPOSED_REPLY, *, finish_reason="stop", refusal=None, usage=(120, 30, 150)): + tokens = FakeUsage(*usage) if usage is not None else None + return FakeResponse([FakeChoice(FakeChatMessage(text, refusal), finish_reason)], tokens) + + +def fake_extract_text(response): + return response.choices[0].message.content + + +class FakeModel: + def __init__(self, deployment, *, provider="aoai", endpoint_id="endpoint-1", model_id=None): + self.deployment = deployment + self.provider = provider + self.behavior_name = deployment + self.model_metadata = None + self.endpoint_id = endpoint_id + self.model_id = model_id or f"model-{deployment}" + self.replies = [] + self.requests = [] + self.closed = 0 + + def metadata(self): + return { + "model_deployment_name": self.deployment, + "model_provider": self.provider, + "model_endpoint_id": self.endpoint_id, + "model_id": self.model_id, + } + + def create_completion(self, messages=None, temperature=None, max_tokens=None): + self.requests.append({"messages": copy.deepcopy(messages), "temperature": temperature, "max_tokens": max_tokens}) + reply = self.replies.pop(0) if self.replies else fake_response() + if isinstance(reply, BaseException): + raise reply + return reply + + def close(self): + self.closed += 1 + + +class FakeModels: + """``resolve_orchestration_model``: the request's selected model, then the default model.""" + + def __init__(self): + self.selected = FakeModel("gpt-selected") + self.default = FakeModel("gpt-default") + self.unavailable = set() + self.calls = [] + + def __call__(self, settings, user_id=None, seeds=None, identity_context=None): + which = "selected" if isinstance(seeds, dict) and seeds.get("model") else "default" + self.calls.append({"which": which, "seeds": copy.deepcopy(seeds), "user_id": user_id, "identity": identity_context}) + if which in self.unavailable: + raise FakeModelError(which) + return self.selected if which == "selected" else self.default + + @property + def requests(self): + return self.selected.requests + self.default.requests + + +class FakeCheck: + def __init__(self, blocked=False, categories=()): + self.blocked = blocked + self.categories = list(categories) + + +class FakeContentChecker: + def __init__(self): + self.calls = [] + self.block = False + self.error = None + + def __call__(self, text, surface, user_id=None, settings=None): + self.calls.append({"text": text, "surface": surface, "user_id": user_id}) + if self.error is not None: + raise self.error + return FakeCheck(blocked=self.block, categories=["hate"] if self.block else []) + + +def fake_attach_chat_check(document, check): + document["content_check"] = { + "blocked": bool(getattr(check, "blocked", False)), + "categories": list(getattr(check, "categories", [])), + } + return document + + +class Recorder: + """A callable that records its calls and raises queued errors first.""" + + def __init__(self, result=None): + self.calls = [] + self.errors = [] + self.result = result + + def __call__(self, *args, **kwargs): + self.calls.append({"args": copy.deepcopy(args), "kwargs": copy.deepcopy(kwargs)}) + if self.errors: + raise self.errors.pop(0) + return self.result + + +class FakeNotifications: + """Both notification helpers. Each idempotency key stores one notice, like the real helpers.""" + + def __init__(self): + self.chat_calls = [] + self.notice_calls = [] + self.stored = {} + self.chat_errors = [] + self.notice_errors = [] + + def chat_response(self, user_id, conversation_id, message_id, conversation_title=None, response_preview=None, + idempotency_key=None, strict=False): + call = { + "type": "chat_response", + "user_id": user_id, + "conversation_id": conversation_id, + "message_id": message_id, + "conversation_title": conversation_title, + "response_preview": response_preview, + "idempotency_key": idempotency_key, + "strict": strict, + } + self.chat_calls.append(call) + if self.chat_errors: + raise self.chat_errors.pop(0) + self.stored.setdefault(idempotency_key, call) + return {"id": f"notice-{len(self.stored)}"} + + def create(self, user_id=None, notification_type=None, title=None, message=None, link_url=None, metadata=None, + idempotency_key=None, strict=False, **kwargs): + call = { + "type": notification_type, + "user_id": user_id, + "title": title, + "message": message, + "link_url": link_url, + "metadata": copy.deepcopy(metadata), + "idempotency_key": idempotency_key, + "strict": strict, + } + self.notice_calls.append(call) + if self.notice_errors: + raise self.notice_errors.pop(0) + self.stored.setdefault(idempotency_key, call) + return {"id": f"notice-{len(self.stored)}"} + + def notices(self, notification_type=None): + return [call for call in self.stored.values() if notification_type is None or call["type"] == notification_type] + + +class FakeLock: + def __init__(self, available=True): + self.available = available + self.acquired = [] + self.released = [] + + def acquire(self, name, seconds): + self.acquired.append((name, seconds)) + return {"id": name, "seconds": seconds} if self.available else None + + def release(self, lock): + self.released.append(lock) + + +class FakeHints: + def __init__(self): + self.queue = [] + self.signals = [] + + def drain(self, limit): + drained = self.queue[:limit] + del self.queue[:limit] + return drained + + def signal(self, user_id, run_id): + self.signals.append((user_id, run_id)) + if (user_id, run_id) not in self.queue: + self.queue.append((user_id, run_id)) + return True + + +class FakeGates: + def __init__(self): + self.workflows = True + self.results = True + self.calls = [] + + def workflows_enabled(self, settings, user_roles=None): + self.calls.append(("workflows", copy.deepcopy(user_roles))) + return self.workflows + + def results_enabled(self, settings, user_roles=None): + self.calls.append(("results", copy.deepcopy(user_roles))) + return self.results + + +class FakeStreamProbe: + def __init__(self): + self.active = False + self.calls = [] + + def __call__(self, user_id, conversation_id, now): + self.calls.append((user_id, conversation_id, now)) + return self.active + + +class FakeMonotonic: + def __init__(self): + self.value = 1000.0 + self.step = 0.0 + + def __call__(self): + self.value += self.step + return self.value + + +WORKFLOW_NAME = "Daily digest" +REQUEST_TEXT = "Run my daily digest and tell me what it found." +SELECTED_MODEL = {"model": {"model_deployment": "gpt-selected"}, "reasoning_effort": "", "active_group_ids": []} +# Mirrors functions_workflow_result_followup.EXCERPT_BUDGET_STEPS; only steps below 48 KiB are used. +EXCERPT_STEPS = (48 * 1024, 24 * 1024, 12 * 1024, 6 * 1024) + + +def make_workflow(**overrides): + workflow = { + "id": WORKFLOW_ID, + "user_id": USER, + "name": WORKFLOW_NAME, + "durable_execution": True, + "definition_revision": "rev-1", + } + workflow.update(overrides) + return workflow + + +def make_conversation(**overrides): + conversation = { + "id": CONVERSATION_ID, + "user_id": USER, + "title": "Planning chat", + "chat_type": "personal", + "last_updated": "2026-05-04T14:05:10.000000", + "has_unread_assistant_response": False, + "last_unread_assistant_message_id": None, + "last_unread_assistant_at": None, + } + conversation.update(overrides) + return conversation + + +def _thread(thread_id): + return {"thread_id": thread_id, "previous_thread_id": None, "active_thread": True, "thread_attempt": 1} + + +def make_request_messages(): + return [ + { + "id": USER_MESSAGE_ID, + "conversation_id": CONVERSATION_ID, + "role": "user", + "content": REQUEST_TEXT, + "timestamp": "2026-05-04T14:05:00.000000", + "metadata": {"thread_info": _thread("thread-request")}, + }, + { + "id": "message-assistant-1", + "conversation_id": CONVERSATION_ID, + "role": "assistant", + "content": "I started Daily digest. I'll post the results here when the run finishes.", + "timestamp": "2026-05-04T14:05:10.000000", + "metadata": {"thread_info": _thread("thread-request")}, + }, + ] + + +def make_invocation(**overrides): + invocation = { + "version": 1, + "source": "chat_orchestration", + "conversation_id": CONVERSATION_ID, + "user_message_id": USER_MESSAGE_ID, + "orchestration_run_id": ORCHESTRATION_RUN_ID, + "attempt_root_run_id": ATTEMPT_ROOT_RUN_ID, + "step_id": STEP_ID, + "requested_by": USER, + "requested_at": REQUESTED_AT, + } + invocation.update(overrides) + return invocation + + +def make_record(*, time_zone="America/New_York", model_selection=None, roles=("User",), deadline_at=None, **overrides): + seed = build_chat_delivery_seed( + time_zone=time_zone, + model_selection=SELECTED_MODEL if model_selection is None else model_selection, + requester_roles=list(roles), + now=parse_iso(REQUESTED_AT), + ) + record = finalize_chat_delivery_seed(seed, {"limits": {"deadline_at": deadline_at}} if deadline_at else None) + record.update(overrides) + return record + + +def make_run(*, status="completed", record=None, **overrides): + run = { + "id": RUN_ID, + "user_id": USER, + "workflow_id": WORKFLOW_ID, + "workflow_name": WORKFLOW_NAME, + "status": status, + "trigger_source": "chat_orchestration", + "definition_revision": "rev-1", + "chat_invocation": make_invocation(), + "chat_delivery": make_record() if record is None else record, + } + run.update(overrides) + return run + + +def newest_message_handler(container, query, params, partition_key): + require("ORDER BY c.timestamp DESC" in query and "TOP 1" in query, f"unexpected message query {query!r}") + conversation_id = params.get("@conversation_id") + require(partition_key == conversation_id, "the newest-message query must stay in the chat's partition") + rows = [item for item in container.items.values() if item.get("conversation_id") == conversation_id] + rows.sort(key=lambda item: str(item.get("timestamp") or ""), reverse=True) + return [ + { + "id": row["id"], + "role": row.get("role"), + "timestamp": row.get("timestamp"), + "thread_info": (row.get("metadata") or {}).get("thread_info"), + } + for row in rows[:1] + ] + + +def _at_or_before(value, moment): + return isinstance(value, str) and value <= moment + + +_SWEEP_NEXT_UNDEFINED = "NOT IS_DEFINED(c.chat_delivery.next_attempt_at)" +_SWEEP_NEXT_NULL = "IS_NULL(c.chat_delivery.next_attempt_at)" +_SWEEP_NEXT_DUE = "c.chat_delivery.next_attempt_at <= @now" +_SWEEP_READY = "c.chat_delivery.status = @ready" +_SWEEP_LEASE = "(c.chat_delivery.status = @delivering AND c.chat_delivery.lease_expires_at <= @now)" +_SWEEP_PENDING = "c.chat_delivery.status = @pending AND (" +_SWEEP_EXPIRY = "c.chat_delivery.expires_at <= @now" +_SWEEP_GRACE = "(ARRAY_CONTAINS(@terminal, c.status) AND c._ts <= @grace_ts)" +_SWEEP_TOP = "SELECT TOP @top " + + +def _next_attempt_due(query, record, now): + """Cosmos semantics: an undefined or null field passes only through its own clause.""" + if not any(clause in query for clause in (_SWEEP_NEXT_UNDEFINED, _SWEEP_NEXT_NULL, _SWEEP_NEXT_DUE)): + return True + if "next_attempt_at" not in record: + return _SWEEP_NEXT_UNDEFINED in query + value = record["next_attempt_at"] + if value is None: + return _SWEEP_NEXT_NULL in query + return _SWEEP_NEXT_DUE in query and _at_or_before(value, now) + + +def sweep_handler(container, query, params, partition_key): + """A Python mirror of the sweep's WHERE clause, so the sweep can be driven end to end. + + Each branch applies only when its clause text is in the query, so a production change that drops + or rewrites a branch changes which runs the fake returns instead of being mirrored away. + """ + require("IS_DEFINED(c.chat_delivery)" in query, f"unexpected run query {query!r}") + now = params["@now"] + rows = [] + for item in container.items.values(): + record = item.get("chat_delivery") + if not isinstance(record, dict) or not _next_attempt_due(query, record, now): + continue + status = record.get("status") + due = ( + (_SWEEP_READY in query and status == params["@ready"]) + or ( + _SWEEP_LEASE in query + and status == params["@delivering"] + and _at_or_before(record.get("lease_expires_at"), now) + ) + or ( + _SWEEP_PENDING in query + and status == params["@pending"] + and ( + (_SWEEP_EXPIRY in query and _at_or_before(record.get("expires_at"), now)) + or ( + _SWEEP_GRACE in query + and item.get("status") in params["@terminal"] + and item.get("_ts", 0) <= params["@grace_ts"] + ) + ) + ) + ) + if due: + rows.append({"id": item["id"], "user_id": item["user_id"]}) + return rows[: params["@top"]] if _SWEEP_TOP in query else rows + + +@dataclass +class DeliveryWorld: + services: object + settings: dict + clock: FakeClock + runs: FakeContainer + workflows: FakeContainer + conversations: FakeContainer + messages: FakeContainer + runtime: FakeRuntime + reader: FakeResultReader + disclosure: FakeDisclosure + prompts: FakePromptBuilder + nonces: FakeNonces + budget: FakeBudget + models: FakeModels + checker: FakeContentChecker + notifications: FakeNotifications + tokens: Recorder + incidents: Recorder + cache_bumps: Recorder + identities: Recorder + lock: FakeLock + hints: FakeHints + gates: FakeGates + stream: FakeStreamProbe + monotonic: FakeMonotonic + unread_calls: list = field(default_factory=list) + + def run(self, run_id=RUN_ID): + return self.runs.get(run_id) + + def record(self, run_id=RUN_ID): + return (self.run(run_id) or {}).get("chat_delivery") + + def conversation(self): + return self.conversations.get(CONVERSATION_ID) + + def delivery_messages(self): + return sorted( + (item for item in self.messages.items.values() if str(item["id"]).startswith("assistant_workflow_delivery_")), + key=lambda item: item["timestamp"], + ) + + def set_record(self, run_id=RUN_ID, **changes): + run = self.runs.items[run_id] + record = dict(run["chat_delivery"]) + record.update(changes) + run["chat_delivery"] = record + self.runs.put(run) + return record + + +def make_world(*, run=None, state="completed", version=5, control=True, settings=None): + clock = FakeClock() + world_settings = {"enable_workflows": True, "enable_chat_workflow_results": True} if settings is None else settings + runtime = FakeRuntime() + if control: + runtime.set(state=state, version=version) + unread_calls = [] + + def mark_unread(container, conversation_id, user_id, message_id, planned_at, *, require_private=True, + skip_if_read_since=None): + unread_calls.append({ + "conversation_id": conversation_id, + "user_id": user_id, + "message_id": message_id, + "planned_at": planned_at, + "require_private": require_private, + "skip_if_read_since": skip_if_read_since, + }) + return mark_conversation_unread_guarded( + container, conversation_id, user_id, message_id, planned_at, + require_private=require_private, skip_if_read_since=skip_if_read_since, + ) + + world = DeliveryWorld( + services=None, + settings=world_settings, + clock=clock, + runs=FakeContainer("runs", [make_run() if run is None else run], query_handler=sweep_handler, clock=clock), + workflows=FakeContainer("workflows", [make_workflow()]), + conversations=FakeContainer("conversations", [make_conversation()]), + messages=FakeContainer("messages", make_request_messages(), query_handler=newest_message_handler), + runtime=runtime, + reader=FakeResultReader(), + disclosure=FakeDisclosure(), + prompts=FakePromptBuilder(), + nonces=FakeNonces(), + budget=FakeBudget(), + models=FakeModels(), + checker=FakeContentChecker(), + notifications=FakeNotifications(), + tokens=Recorder(), + incidents=Recorder(), + cache_bumps=Recorder(), + identities=Recorder(result={"identity": "captured"}), + lock=FakeLock(), + hints=FakeHints(), + gates=FakeGates(), + stream=FakeStreamProbe(), + monotonic=FakeMonotonic(), + unread_calls=unread_calls, + ) + world.services = WorkflowChatDeliveryServices( + runs=world.runs, + workflows=world.workflows, + conversations=world.conversations, + messages=world.messages, + get_settings=lambda: dict(world.settings), + runtime_store_factory=runtime.factory, + runtime_conflict_error=FakeRuntimeConflict, + runtime_unavailable_error=FakeRuntimeUnavailable, + read_workflow_result=world.reader, + result_unavailable_error=FakeResultUnavailable, + workflow_result_context=fake_workflow_result_context, + format_run_time=fake_format_run_time, + format_disclosure=world.disclosure, + build_result_messages=world.prompts, + new_fence_nonce=world.nonces, + excerpt_budget_steps=EXCERPT_STEPS, + calculate_budget=world.budget, + resolve_model=world.models, + model_error=FakeModelError, + capture_identity=world.identities, + extract_text=fake_extract_text, + check_content=world.checker, + attach_chat_check=fake_attach_chat_check, + record_incident=world.incidents, + is_message_retracted=lambda message: bool(message.get("retracted")), + create_chat_response_notification=world.notifications.chat_response, + create_notification=world.notifications.create, + log_token_usage=world.tokens, + bump_cache_version=world.cache_bumps, + mark_unread_guarded=mark_unread, + is_conversation_private=conversation_is_private, + is_user_workflows_enabled=world.gates.workflows_enabled, + is_chat_workflow_results_enabled=world.gates.results_enabled, + stream_activity_probe=world.stream, + acquire_lock=world.lock.acquire, + release_lock=world.lock.release, + drain_hints=world.hints.drain, + signal_delivery=world.hints.signal, + clock=clock, + compose_output_tokens=None, + monotonic=world.monotonic, + ) + return world diff --git a/functional_tests/test_support/workflow_chat_delivery_off_golden.json b/functional_tests/test_support/workflow_chat_delivery_off_golden.json new file mode 100644 index 000000000..32ce05667 --- /dev/null +++ b/functional_tests/test_support/workflow_chat_delivery_off_golden.json @@ -0,0 +1,332 @@ +{ + "answer_and_run": { + "first_write": [ + { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + ], + "message": { + "_etag": "<_etag>", + "agent_citations": [], + "augmented": false, + "content": "Your priorities.\n\nSaved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "conversation_id": "conversation-1", + "generated_artifacts": [], + "hybrid_citations": [], + "id": "assistant_orchestration_e4eec6448d10579e9dc3ef181f1393b1db184560", + "metadata": { + "orchestration": { + "attempt_index": 1, + "failure": null, + "failures": [], + "finalization_status": "saved", + "message_saved": true, + "outcome": "completed", + "outputs": [], + "plan_summary": { + "capabilities_used": [ + "compose", + "workflow_run" + ], + "intent_summary": "", + "plan_id": "plan-1", + "run_id": "run-1", + "status": "completed", + "step_count": 2, + "turn_id": "turn-1" + }, + "recovery": { + "eligible": false, + "expected_version": "", + "message": "This request is already complete.", + "reason_code": "already_completed", + "requires_confirmation": false, + "retry_step_ids": [], + "reused_step_ids": [ + "answer", + "run_digest" + ], + "source_run_id": "run-1" + }, + "retry_of_run_id": null, + "run_id": "run-1", + "status": "completed", + "turn_id": "turn-1" + }, + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "token_usage": { + "completion_tokens": 3, + "prompt_tokens": 7, + "total_tokens": 10 + } + }, + "model_deployment_name": "gpt-4o", + "model_provider": "aoai", + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "role": "assistant", + "timestamp": "", + "web_search_citations": [] + }, + "notes": { + "one": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "stopped": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.\n\nStopping this plan doesn't stop a workflow it already started. Cancel the run in Workflows if you need to.", + "two": "Saved workflows:\n- Started `Weekly digest`.\n- Started `Other digest`.\n\nFollow each run's progress and results in that workflow's run history in Workflows. Results also appear wherever each workflow already sends them, such as its conversation or alerts." + }, + "queued": [ + { + "actor_user_id": "owner", + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "request_id": "023264b5-a9ca-5c85-a03e-f351f8b7f984", + "trigger_source": "chat_orchestration" + } + ], + "sidecar": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "handle": "workflow-weekly-digest-702b6c", + "name": "Weekly digest", + "orchestration_run_id": "run-1", + "reason": null, + "requested_by": "owner", + "run_id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "status": "queued", + "step_id": "run_digest", + "version": 1, + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001" + }, + "step_summary": "Started the saved workflow.", + "stored_run": { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + }, + "run_only": { + "first_write": [ + { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + ], + "message": { + "_etag": "<_etag>", + "agent_citations": [], + "augmented": false, + "content": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "conversation_id": "conversation-1", + "generated_artifacts": [], + "hybrid_citations": [], + "id": "assistant_orchestration_e4eec6448d10579e9dc3ef181f1393b1db184560", + "metadata": { + "orchestration": { + "attempt_index": 1, + "failure": null, + "failures": [], + "finalization_status": "saved", + "message_saved": true, + "outcome": "completed", + "outputs": [], + "plan_summary": { + "capabilities_used": [ + "workflow_run" + ], + "intent_summary": "", + "plan_id": "plan-1", + "run_id": "run-1", + "status": "completed", + "step_count": 1, + "turn_id": "turn-1" + }, + "recovery": { + "eligible": false, + "expected_version": "", + "message": "This request is already complete.", + "reason_code": "already_completed", + "requires_confirmation": false, + "retry_step_ids": [], + "reused_step_ids": [ + "run_digest" + ], + "source_run_id": "run-1" + }, + "retry_of_run_id": null, + "run_id": "run-1", + "status": "completed", + "turn_id": "turn-1" + }, + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "token_usage": { + "completion_tokens": 0, + "prompt_tokens": 0, + "total_tokens": 0 + } + }, + "model_deployment_name": "gpt-4o", + "model_provider": "aoai", + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "role": "assistant", + "timestamp": "", + "web_search_citations": [] + }, + "notes": { + "one": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "stopped": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.\n\nStopping this plan doesn't stop a workflow it already started. Cancel the run in Workflows if you need to.", + "two": "Saved workflows:\n- Started `Weekly digest`.\n- Started `Other digest`.\n\nFollow each run's progress and results in that workflow's run history in Workflows. Results also appear wherever each workflow already sends them, such as its conversation or alerts." + }, + "queued": [ + { + "actor_user_id": "owner", + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "request_id": "023264b5-a9ca-5c85-a03e-f351f8b7f984", + "trigger_source": "chat_orchestration" + } + ], + "sidecar": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "handle": "workflow-weekly-digest-702b6c", + "name": "Weekly digest", + "orchestration_run_id": "run-1", + "reason": null, + "requested_by": "owner", + "run_id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "status": "queued", + "step_id": "run_digest", + "version": 1, + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001" + }, + "step_summary": "Started the saved workflow.", + "stored_run": { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + } +} diff --git a/functional_tests/test_workflow_chat_delivery_concurrency.py b/functional_tests/test_workflow_chat_delivery_concurrency.py new file mode 100644 index 000000000..8c822686d --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_concurrency.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_concurrency.py +""" +Functional test for workflow chat delivery concurrency. +Version: 0.261.227 +Implemented in: 0.261.227 + +This test ensures concurrent chat delivery workers publish each workflow result exactly once. +""" + +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +import functions_workflow_chat_delivery_worker as worker # noqa: E402 +from functions_workflow_chat_delivery import ( # noqa: E402 + LEASE_SECONDS, + RECENT_USER_MESSAGE_SECONDS, + clear_workflow_chat_delivery_hints, + workflow_delivery_message_id, +) +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + RUN_ID, + USER, + make_world, + require, +) + + +M5 = workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) +THREAD_TIMEOUT_SECONDS = 8.0 +RENDEZVOUS_TIMEOUT_SECONDS = 5.0 + + +@pytest.fixture(autouse=True) +def clear_delivery_hints(): + clear_workflow_chat_delivery_hints() + yield + clear_workflow_chat_delivery_hints() + + +def deliver(world): + return worker.process_workflow_chat_delivery(USER, RUN_ID, services=world.services) + + +def _synchronize_container_writes(monkeypatch, *containers): + for container in containers: + write_lock = threading.Lock() + for method_name in ("replace_item", "create_item", "upsert_item", "delete_item"): + original = getattr(container, method_name) + + def synchronized(*args, _original=original, _lock=write_lock, **kwargs): + with _lock: + return _original(*args, **kwargs) + + monkeypatch.setattr(container, method_name, synchronized) + + +def _barrier_first_run_reads(monkeypatch, world): + barrier = threading.Barrier(2, timeout=RENDEZVOUS_TIMEOUT_SECONDS) + original = world.runs.read_item + seen_threads = set() + seen_lock = threading.Lock() + + def read_item(*args, **kwargs): + thread_id = threading.get_ident() + with seen_lock: + first_for_thread = thread_id not in seen_threads + if first_for_thread: + seen_threads.add(thread_id) + item = original(*args, **kwargs) + if first_for_thread: + barrier.wait() + return item + + monkeypatch.setattr(world.runs, "read_item", read_item) + + +def _patch_compose_rendezvous(monkeypatch, world): + condition = threading.Condition() + arrivals = {"count": 0} + + def arrive(): + with condition: + arrivals["count"] += 1 + condition.notify_all() + deadline = time.monotonic() + RENDEZVOUS_TIMEOUT_SECONDS + while arrivals["count"] < 2: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AssertionError("timed out waiting for the other delivery worker") + condition.wait(remaining) + + def returned(): + with condition: + arrivals["count"] += 1 + condition.notify_all() + + for model in (world.models.selected, world.models.default): + original = model.create_completion + + def create_completion(*args, _original=original, **kwargs): + arrive() + return _original(*args, **kwargs) + + monkeypatch.setattr(model, "create_completion", create_completion) + + return returned + + +def _run_threads(actions, *, on_return=None): + results = {} + errors = {} + + def target(name, action): + try: + results[name] = action() + except Exception as exc: + errors[name] = exc + finally: + if on_return is not None: + on_return() + + threads = [ + threading.Thread(target=target, args=(name, action), daemon=True) + for name, action in actions.items() + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(THREAD_TIMEOUT_SECONDS) + + unfinished = [thread.name for thread in threads if thread.is_alive()] + require(not unfinished, f"delivery threads did not finish: {unfinished}") + if errors: + name, error = next(iter(errors.items())) + raise AssertionError(f"delivery thread {name} failed") from error + silent = [name for name in actions if name not in results] + require(not silent, f"delivery threads ended without a result: {silent}") + return results + + +def _require_one_delivery_side_effect(world): + messages = world.delivery_messages() + message_ids = [message["id"] for message in messages] + require(message_ids == [M5], f"expected exactly one delivery message, got {message_ids}") + chat_notice_count = len(world.notifications.chat_calls) + require(chat_notice_count == 1, f"expected one chat notice, got {chat_notice_count}") + unread_count = len(world.unread_calls) + require(unread_count == 1, f"expected one unread mark, got {unread_count}") + model_request_count = len(world.models.requests) + require(model_request_count == 1, f"expected one model request, got {model_request_count}") + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.227") + + +def test_claim_race_allows_one_worker_to_deliver_and_the_other_to_stay_busy(monkeypatch): + world = make_world() + _synchronize_container_writes(monkeypatch, world.runs, world.messages, world.conversations) + _barrier_first_run_reads(monkeypatch, world) + mark_worker_returned = _patch_compose_rendezvous(monkeypatch, world) + + results = _run_threads( + { + "worker-a": lambda: deliver(world), + "worker-b": lambda: deliver(world), + }, + on_return=mark_worker_returned, + ) + + outcomes = set(results.values()) + expected_outcomes = {worker.OUTCOME_DELIVERED, worker.OUTCOME_BUSY} + require( + outcomes == expected_outcomes, + "two workers racing for one claim: exactly one delivers and the other stays busy; " + f"got {dict(sorted(results.items()))}", + ) + _require_one_delivery_side_effect(world) + record = world.record() + status = record.get("status") + attempts = record.get("attempts") + require(status == "delivered", f"expected delivered record, got {status}") + require(attempts == 1, f"expected one attempt, got {attempts}") + + +def test_hint_and_sweep_race_delivers_once(monkeypatch): + world = make_world() + world.clock.advance(minutes=3) + world.hints.signal(USER, RUN_ID) + _synchronize_container_writes(monkeypatch, world.runs, world.messages, world.conversations) + _barrier_first_run_reads(monkeypatch, world) + + results = _run_threads({ + "hint": lambda: worker.process_workflow_chat_delivery_hints(world.services), + "sweep": lambda: worker.run_workflow_chat_delivery_sweep(world.services), + }) + + hint_outcomes = results["hint"] + sweep_summary = results["sweep"] + require(len(hint_outcomes) == 1, f"expected one hint outcome, got {hint_outcomes}") + require(sweep_summary["locked"] is True, f"expected locked sweep, got {sweep_summary}") + require(sweep_summary["processed"] == 1, f"expected one sweep row, got {sweep_summary}") + _require_one_delivery_side_effect(world) + record = world.record() + status = record.get("status") + attempts = record.get("attempts") + require(status == "delivered", f"expected delivered record, got {status}") + require(attempts == 1, f"expected one attempt, got {attempts}") + + +def test_stale_worker_loses_lease_after_reclaimed_delivery(monkeypatch): + world = make_world() + _synchronize_container_writes(monkeypatch, world.runs, world.messages, world.conversations) + original = world.models.selected.create_completion + reentered = {"done": False} + reclaimed_outcomes = [] + + def create_completion(*args, **kwargs): + if not reentered["done"]: + reentered["done"] = True + seconds = max(LEASE_SECONDS, RECENT_USER_MESSAGE_SECONDS) + 1 + world.clock.advance(seconds=seconds) + reclaimed_outcomes.append(deliver(world)) + return original(*args, **kwargs) + + monkeypatch.setattr(world.models.selected, "create_completion", create_completion) + + stale_outcome = deliver(world) + + require(reclaimed_outcomes == [worker.OUTCOME_DELIVERED], f"unexpected reclaimed outcomes: {reclaimed_outcomes}") + require(stale_outcome == worker.OUTCOME_LOST_LEASE, f"expected lost lease, got {stale_outcome}") + messages = world.delivery_messages() + message_ids = [message["id"] for message in messages] + require(message_ids == [M5], f"expected one delivery message, got {message_ids}") + chat_notice_count = len(world.notifications.chat_calls) + require(chat_notice_count == 1, f"expected one chat notice, got {chat_notice_count}") + unread_count = len(world.unread_calls) + require(unread_count == 1, f"expected one unread mark, got {unread_count}") + record = world.record() + status = record.get("status") + lease_id = record.get("lease_id") + lease_expires_at = record.get("lease_expires_at") + attempts = record.get("attempts") + require(status == "delivered", f"expected delivered record, got {status}") + require(lease_id is None, f"expected cleared lease id, got {lease_id}") + require(lease_expires_at is None, f"expected cleared lease expiry, got {lease_expires_at}") + require(attempts == 2, f"expected two attempts, got {attempts}") + + +def test_two_sequential_sweeps_after_delivery_do_not_repeat_side_effects(): + world = make_world() + world.clock.advance(minutes=3) + + first_summary = worker.run_workflow_chat_delivery_sweep(world.services) + first_messages = world.delivery_messages() + first_message_count = len(first_messages) + first_notice_count = len(world.notifications.chat_calls) + first_unread_count = len(world.unread_calls) + first_model_count = len(world.models.requests) + + second_summary = worker.run_workflow_chat_delivery_sweep(world.services) + second_messages = world.delivery_messages() + second_message_count = len(second_messages) + second_notice_count = len(world.notifications.chat_calls) + second_unread_count = len(world.unread_calls) + second_model_count = len(world.models.requests) + + require(first_summary["outcomes"] == {worker.OUTCOME_DELIVERED: 1}, f"unexpected first sweep: {first_summary}") + require(second_summary["processed"] == 0, f"unexpected second sweep: {second_summary}") + require(first_message_count == 1, f"expected first sweep to create one message, got {first_message_count}") + require(second_message_count == first_message_count, "second sweep created another delivery message") + require(second_notice_count == first_notice_count, "second sweep sent another chat notice") + require(second_unread_count == first_unread_count, "second sweep marked unread again") + require(second_model_count == first_model_count, "second sweep made another model request") + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) diff --git a/functional_tests/test_workflow_chat_delivery_contract.py b/functional_tests/test_workflow_chat_delivery_contract.py new file mode 100644 index 000000000..c240757cd --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_contract.py @@ -0,0 +1,836 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_contract.py +""" +Functional test for the workflow chat delivery contract module. +Version: 0.261.227 +Implemented in: 0.261.227 + +This test ensures the pure delivery helpers preserve idempotent chat-result message contracts. +""" + +import copy +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +import functions_workflow_chat_delivery as delivery # noqa: E402 +from functions_workflow_result_reader import format_workflow_run_time # noqa: E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + RUN_ID, + STEP_ID, + USER, + WORKFLOW_ID, + require, +) + +NOW = datetime(2026, 5, 4, 15, 0, 0, tzinfo=timezone.utc) +NOW_TEXT = "2026-05-04T15:00:00.000000Z" +REQUESTED_AT = "2026-05-04T14:05:00.000000Z" + + +def _summary(state="completed", version=5, **overrides): + summary = { + "exists": True, + "deleted": False, + "version": version, + "state": state, + "phase": None, + "gate_reason_code": None, + "deadline_at": None, + "deadline_seconds": None, + "schema_version": 2, + } + summary.update(overrides) + return summary + + +def _record(status=delivery.STATUS_PENDING, generation=None, **overrides): + record = delivery.build_chat_delivery_seed( + time_zone="America/New_York", + model_selection={"model": {"model_deployment": "gpt-4o"}, "reasoning_effort": "medium"}, + requester_roles=["User"], + now=NOW, + ) + record = delivery.finalize_chat_delivery_seed(record, None) + record.update({"status": status, "generation": generation, "updated_at": NOW_TEXT}) + record.update(overrides) + return record + + +def _reconcile(record, summary, now=NOW): + return delivery.reconcile_chat_delivery(record, summary, now=now) + + +@pytest.fixture(autouse=True) +def clear_delivery_hints(): + delivery.clear_workflow_chat_delivery_hints() + yield + delivery.clear_workflow_chat_delivery_hints() + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.227") + + +def test_workflow_delivery_message_id_is_deterministic_and_prefixed(): + first = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + second = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + require(first == second, "message id should be deterministic for the same run, chat, and generation") + require(first.startswith(delivery.DELIVERY_MESSAGE_ID_PREFIX), "message id should carry the delivery prefix") + require(len(first) == len(delivery.DELIVERY_MESSAGE_ID_PREFIX) + 40, "message id should use a 40-char fingerprint") + + +def test_workflow_delivery_message_id_changes_when_generation_changes(): + old_id = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + new_id = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 6) + require(old_id != new_id, "message id must include generation in its fingerprint") + + +def test_workflow_delivery_message_id_changes_when_run_or_conversation_changes(): + first = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + other_run = delivery.workflow_delivery_message_id(f"{RUN_ID}-other", CONVERSATION_ID, 5) + other_conversation = delivery.workflow_delivery_message_id(RUN_ID, f"{CONVERSATION_ID}-other", 5) + require(first != other_run, "message id should change when only the run id changes") + require(first != other_conversation, "message id should change when only the conversation id changes") + + +def test_workflow_delivery_message_id_rejects_missing_inputs(): + with pytest.raises(ValueError, match="run_id is required"): + delivery.workflow_delivery_message_id("", CONVERSATION_ID, 5) + with pytest.raises(ValueError, match="conversation_id is required"): + delivery.workflow_delivery_message_id(RUN_ID, "", 5) + with pytest.raises(ValueError, match="generation must be a non-negative integer"): + delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, -1) + with pytest.raises(ValueError, match="generation must be a non-negative integer"): + delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, True) + + +def test_delivery_thread_id_is_deterministic_and_distinct_per_message(): + first_message = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + second_message = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 6) + first_thread = delivery.delivery_thread_id(first_message) + repeat_thread = delivery.delivery_thread_id(first_message) + second_thread = delivery.delivery_thread_id(second_message) + require(first_thread == repeat_thread, "thread id should be deterministic for one message id") + require(first_thread != second_thread, "different message ids should get different thread ids") + with pytest.raises(ValueError, match="message_id is required"): + delivery.delivery_thread_id(" ") + + +def test_is_workflow_delivery_message_accepts_metadata_and_delivery_id(): + metadata_message = {"metadata": {delivery.DELIVERY_METADATA_KEY: {"run_id": RUN_ID}}} + id_message = {"id": delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5), "metadata": {}} + require(delivery.is_workflow_delivery_message(metadata_message), "metadata key should identify a workflow delivery message") + require(delivery.is_workflow_delivery_message(id_message), "delivery id prefix should identify a workflow delivery message") + + +def test_is_workflow_delivery_message_rejects_ordinary_and_malformed_messages(): + samples = [ + None, + [], + {}, + {"id": "assistant_regular_1", "metadata": {}}, + {"metadata": None}, + {"metadata": "not-a-dict"}, + {"metadata": {delivery.DELIVERY_METADATA_KEY: "not-a-dict"}}, + ] + for sample in samples: + result = delivery.is_workflow_delivery_message(sample) + require(result is False, f"malformed message should not be recognized: {sample!r}") + + +def test_delivery_notification_key_formats_are_pinned_literals(): + key = delivery.delivery_notification_key(RUN_ID, 5) + notice_key = delivery.undeliverable_notification_key(RUN_ID, 5) + expired_key = delivery.expired_notification_key(RUN_ID) + require(key == "workflow-chat-delivery:run-delivery-1:5", "delivery notification key format changed") + require(notice_key == "workflow-chat-delivery-notice:run-delivery-1:5", "undeliverable notification key format changed") + require(expired_key == "workflow-chat-delivery-notice:run-delivery-1:expired", "expired notification key format changed") + + +def test_token_usage_idempotency_key_format_is_pinned_literal(): + message_id = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + key = delivery.token_usage_idempotency_key(message_id, 3) + require(key == f"workflow_result_delivery:{message_id}:3", "token usage idempotency key format changed") + + +def test_idempotency_keys_differ_by_generation_and_attempt(): + gen_five = delivery.delivery_notification_key(RUN_ID, 5) + gen_six = delivery.delivery_notification_key(RUN_ID, 6) + attempt_one = delivery.token_usage_idempotency_key("message-1", 1) + attempt_two = delivery.token_usage_idempotency_key("message-1", 2) + require(gen_five != gen_six, "notification keys must vary by generation") + require(attempt_one != attempt_two, "token usage keys must vary by attempt") + + +def test_format_delivery_timestamp_normalizes_naive_and_aware_datetimes(): + naive = datetime(2026, 5, 4, 15, 0, 1, 123456) + aware = datetime(2026, 5, 4, 11, 0, 1, 123456, tzinfo=timezone(timedelta(hours=-4))) + require(delivery.format_delivery_timestamp(naive) == "2026-05-04T15:00:01.123456Z", "naive timestamps should be treated as UTC") + require(delivery.format_delivery_timestamp(aware) == "2026-05-04T15:00:01.123456Z", "aware timestamps should normalize to UTC") + require(delivery.format_delivery_timestamp("not-datetime") is None, "non-datetime values should not format") + + +def test_parse_delivery_timestamp_round_trips_and_normalizes(): + original = datetime(2026, 5, 4, 11, 0, 1, tzinfo=timezone(timedelta(hours=-4))) + formatted = delivery.format_delivery_timestamp(original) + parsed = delivery.parse_delivery_timestamp(formatted) + require(parsed == datetime(2026, 5, 4, 15, 0, 1, tzinfo=timezone.utc), "format/parse should round trip through UTC") + naive = delivery.parse_delivery_timestamp("2026-05-04T15:00:01") + require(naive == datetime(2026, 5, 4, 15, 0, 1, tzinfo=timezone.utc), "naive parsed timestamps should become UTC") + + +def test_parse_delivery_timestamp_returns_none_for_junk_inputs(): + for value in ("", "not-a-date", "1" * 65, object(), None): + parsed = delivery.parse_delivery_timestamp(value) + require(parsed is None, f"junk timestamp should parse as None: {value!r}") + + +def test_delivery_label_includes_quoted_name_and_asked_when(): + label = delivery.delivery_label("Daily digest", "Sun Jan 4, 2026, 9:30 PM EST") + without_time = delivery.delivery_label("Daily digest", "") + require(label == "Results from `Daily digest` ยท you asked on Sun Jan 4, 2026, 9:30 PM EST", "delivery label format changed") + require(without_time == "Results from `Daily digest`", "empty asked time should be omitted from delivery label") + + +def test_format_workflow_run_time_pins_asked_time_in_new_york(): + rendered = format_workflow_run_time("2026-01-05T02:30:00Z", "America/New_York") + require(rendered == "Sun Jan 4, 2026, 9:30 PM EST", "asked time format for America/New_York changed") + + +def test_format_workflow_run_time_falls_back_to_utc_for_bad_or_missing_zone(): + bad_zone = format_workflow_run_time("2026-01-05T02:30:00Z", "Not/A_Real_Zone") + missing_zone = format_workflow_run_time("2026-01-05T02:30:00Z", None) + expected = "Mon Jan 5, 2026, 2:30 AM UTC" + require(bad_zone == expected, "invalid time zone should fall back to UTC") + require(missing_zone == expected, "missing time zone should fall back to UTC") + + +def test_clean_catalog_text_drops_noise_collapses_space_and_caps_length(): + cleaned = delivery.clean_catalog_text(" Alpha\u200b\n\tBeta ", 80) + capped = delivery.clean_catalog_text("abcdefghij", 6) + require(cleaned == "Alpha Beta", "catalog text should remove controls and collapse whitespace") + require(capped == "abcdeโ€ฆ", "catalog text should cap with an ellipsis") + require(delivery.clean_catalog_text(42, 80) == "", "non-string catalog text should become empty") + + +def test_quoted_workflow_name_neutralizes_backticks_noise_and_empty_names(): + quoted = delivery.quoted_workflow_name("Bad`Name\u200b\nNext") + empty = delivery.quoted_workflow_name("```\u200b") + require(quoted == "`Bad'Name Next`", "quoted workflow name should neutralize markdown/control characters") + require(empty is None, "names with nothing but backticks and noise should be omitted") + + +def test_quoted_workflow_name_caps_to_name_max_length(): + quoted = delivery.quoted_workflow_name("A" * 90) + inner = quoted.strip("`") + require(len(inner) == delivery.NAME_MAX_LENGTH, "quoted workflow name should cap inner text at NAME_MAX_LENGTH") + require(inner.endswith("โ€ฆ"), "capped workflow names should end with an ellipsis") + + +def test_display_workflow_name_falls_back_to_default_workflow(): + require(delivery.display_workflow_name("Daily digest") == "`Daily digest`", "display name should quote real names") + require(delivery.display_workflow_name("```") == "`Workflow`", "display name should fall back for empty sanitized names") + + +def test_notice_workflow_name_uses_quotes_and_default_text(): + named = delivery.notice_workflow_name('Daily "digest"') + fallback = delivery.notice_workflow_name("\u200b") + require(named == '"Daily \'digest\'"', "notice workflow name should quote and neutralize double quotes") + require(fallback == "your workflow", "notice workflow name should fall back to your workflow") + + +def test_notice_title_pins_every_supported_kind(): + expected = { + delivery.KIND_RESULT: 'Results from "Daily digest" are ready', + delivery.KIND_ANALYSIS: 'Results from "Daily digest" are ready', + delivery.KIND_STATUS: 'Results from "Daily digest" are ready', + delivery.KIND_CONTENT_BLOCKED: 'Results from "Daily digest" are ready', + delivery.KIND_FAILED: '"Daily digest" didn\'t finish', + delivery.KIND_CANCELLED: '"Daily digest" was cancelled', + delivery.KIND_SKIPPED: '"Daily digest" didn\'t run', + delivery.KIND_EXPIRED: '"Daily digest" didn\'t finish in time to post to chat', + } + for kind, title in expected.items(): + actual = delivery.notice_title(kind, "Daily digest") + require(actual == title, f"notice title changed for {kind}") + fallback = delivery.notice_title(delivery.KIND_RESULT, "\u200b") + require(fallback == "Results from your workflow are ready", "default notice title sentence casing changed") + + +def test_delivered_notice_preview_is_pinned(): + preview = delivery.delivered_notice_preview("Daily digest") + fallback = delivery.delivered_notice_preview("\u200b") + require(preview == 'Results from "Daily digest" are in your chat', "delivered preview changed") + require(fallback == "Results from your workflow are in your chat", "default delivered preview changed") + + +def test_assemble_delivery_content_joins_non_empty_parts_with_blank_lines(): + content = delivery.assemble_delivery_content("Label", " Body\n", "Disclosure") + body_only = delivery.assemble_delivery_content("", " Body ", "") + require(content == "Label\n\nBody\n\nDisclosure", "delivery content layout changed") + require(body_only == "Body", "empty label/disclosure should be omitted") + + +def test_failure_reason_text_pins_all_known_codes_and_unknown_default(): + expected = { + "failed": "the run stopped before it finished", + "invalid": "the workflow couldn't run as defined", + "incomplete": "some of its steps didn't finish", + "skipped": "no new or changed files were detected", + delivery.REASON_DEADLINE_EXCEEDED: "it reached its time limit", + "execution_budget_exceeded": "it reached its execution limit", + "repeat_iteration_limit": "it reached its repeat limit", + "m365_authorization": "Microsoft 365 access needs to be reconnected", + "authorization": "your workflow access must be restored", + } + require(set(expected) == set(delivery.FAILURE_REASON_CODES), "failure reason code inventory changed") + for code, text in expected.items(): + actual = delivery.failure_reason_text(code) + require(actual == text, f"failure reason text changed for {code}") + require(delivery.failure_reason_text("new-code") == expected["failed"], "unknown reason codes should use failed text") + + +def test_delivery_note_text_pins_supported_kinds_and_reason_codes(): + expected = { + delivery.KIND_FAILED: "`Daily digest` failed: it reached its time limit.", + delivery.KIND_CANCELLED: "`Daily digest` was cancelled.", + delivery.KIND_STATUS: "`Daily digest` finished. Open the run to see its results.", + delivery.KIND_CONTENT_BLOCKED: "`Daily digest` finished, but its results can't be shown here. Open the run to see them.", + delivery.KIND_ANALYSIS: "`Daily digest` finished with a saved analysis. Ask a follow-up question about it here.", + delivery.KIND_SKIPPED: "`Daily digest` didn't run: no new or changed files were detected.", + } + for kind, text in expected.items(): + reason = delivery.REASON_DEADLINE_EXCEEDED if kind == delivery.KIND_FAILED else None + actual = delivery.delivery_note_text(kind, "Daily digest", reason_code=reason) + require(actual == text, f"delivery note text changed for {kind}") + require(bool(actual), f"delivery note text should not be empty for {kind}") + + +def test_delivery_note_text_rejects_unsupported_expired_kind(): + with pytest.raises(ValueError, match="unsupported delivery note kind"): + delivery.delivery_note_text(delivery.KIND_EXPIRED, "Daily digest") + + +def test_delivery_log_swallows_logging_failures(monkeypatch): + """delivery_log must never let telemetry failures escape.""" + + def broken_log_event(*args, **kwargs): + raise RuntimeError("telemetry unavailable") + + monkeypatch.setattr(delivery, "log_event", broken_log_event) + + result = delivery.delivery_log("logging failure is non-fatal") + + require(result is None, f"delivery_log should swallow log_event failures, got {result!r}") + + +def test_build_delivery_metadata_shape_and_values_are_pinned(): + metadata = delivery.build_delivery_metadata( + kind=delivery.KIND_RESULT, + workflow_id=WORKFLOW_ID, + run_id=RUN_ID, + generation=5, + run_status="completed", + orchestration_run_id="orchestration-run-1", + step_id=STEP_ID, + requested_at=REQUESTED_AT, + ) + expected = { + "version": delivery.CHAT_DELIVERY_VERSION, + "kind": delivery.KIND_RESULT, + "workflow_id": WORKFLOW_ID, + "workflow_scope": delivery.WORKFLOW_SCOPE, + "run_id": RUN_ID, + "generation": 5, + "run_status": "completed", + "orchestration_run_id": "orchestration-run-1", + "step_id": STEP_ID, + "requested_at": REQUESTED_AT, + } + require(metadata == expected, "delivery metadata snapshot shape changed") + + +def test_notice_metadata_shape_and_values_are_pinned(): + metadata = delivery.notice_metadata(WORKFLOW_ID, RUN_ID, delivery.STATUS_UNDELIVERABLE) + require(metadata == { + "workflow_id": WORKFLOW_ID, + "run_id": RUN_ID, + "workflow_scope": delivery.WORKFLOW_SCOPE, + "delivery_status": delivery.STATUS_UNDELIVERABLE, + }, "notice metadata shape changed") + + +def test_workflow_run_notice_link_url_encodes_odd_ids(): + link = delivery.workflow_run_notice_link("workflow id/&?", "run id/&?") + require( + link == "/workflow-activity?workflowId=workflow+id%2F%26%3F&runId=run+id%2F%26%3F&scope=personal", + "workflow run notice link encoding changed", + ) + + +def test_workflow_delivery_refusal_payloads_are_pinned(): + retry_payload, retry_status = delivery.workflow_delivery_refusal_payload(delivery.DELIVERY_RETRY_UNSUPPORTED) + edit_payload, edit_status = delivery.workflow_delivery_refusal_payload(delivery.DELIVERY_EDIT_UNSUPPORTED) + require(retry_status == 400 and edit_status == 400, "refusal status should stay 400") + require(retry_payload == { + "error": "A workflow run posted this message, so it can't be retried here. To run the workflow again, open the run in Workflows.", + "code": delivery.DELIVERY_RETRY_UNSUPPORTED, + }, "retry refusal payload changed") + require(edit_payload == { + "error": "A workflow run posted this message, so it can't be edited. Ask a new question instead.", + "code": delivery.DELIVERY_EDIT_UNSUPPORTED, + }, "edit refusal payload changed") + payload, status = delivery.workflow_delivery_refusal_payload("bogus") + require(status == 400, "unknown refusal status should stay 400") + require(payload["code"] == delivery.DELIVERY_RETRY_UNSUPPORTED, "unknown refusal code should fall back to retry") + + +def test_next_backoff_seconds_follows_table_and_clamps_edges(): + attempts = [-10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 99, "2"] + expected = [30, 30, 30, 60, 120, 240, 480, 900, 900, 900, 900, 30] + actual = [delivery.next_backoff_seconds(value) for value in attempts] + require(actual == expected, "backoff table or clamping changed") + + +def test_normalize_requester_roles_dedupes_preserves_order_and_drops_bad_values(): + roles = ["User", "", None, "Admin", "User", "x" * 65, "Approver"] + normalized = delivery.normalize_requester_roles(roles) + require(normalized == ["User", "Admin", "Approver"], "role normalization should drop invalid roles and dedupe in order") + require(delivery.normalize_requester_roles("User") == [], "non-list roles should normalize to empty") + many_roles = [f"role-{index}" for index in range(delivery.MAX_ROLES + 3)] + capped = delivery.normalize_requester_roles(many_roles) + require(len(capped) == delivery.MAX_ROLES, "role normalization should cap the number of roles") + require(capped[-1] == f"role-{delivery.MAX_ROLES - 1}", "role normalization should preserve first MAX_ROLES entries") + + +def test_normalize_model_selection_keeps_valid_model_reasoning_and_groups(): + seeds = { + "model": { + "model_deployment": " gpt-4o ", + "model_id": "model-1", + "model_endpoint_id": "endpoint-1", + "model_provider": "aoai", + }, + "reasoning_effort": " high ", + "active_group_ids": [" group-1 ", "group-2", "group-1", ""], + } + normalized = delivery.normalize_model_selection(seeds) + require(normalized == { + "model": { + "model_deployment": "gpt-4o", + "model_id": "model-1", + "model_endpoint_id": "endpoint-1", + "model_provider": "aoai", + }, + "reasoning_effort": "high", + "active_group_ids": ["group-1", "group-2"], + }, "model selection normalization changed for valid seeds") + + +def test_normalize_model_selection_drops_invalid_model_and_uses_fallback_groups(): + seeds = {"model": {"model_deployment": 123}, "reasoning_effort": "x" * 40} + normalized = delivery.normalize_model_selection(seeds, active_group_ids=["g1", "g1", "g2"]) + require(normalized == {"model": {}, "reasoning_effort": "", "active_group_ids": ["g1", "g2"]}, "invalid model fields should clear model and use fallback groups") + groups = [f"g-{index}" for index in range(delivery.MAX_GROUPS + 3)] + capped = delivery.normalize_model_selection({}, active_group_ids=groups) + require(capped["active_group_ids"] == groups[:delivery.MAX_GROUPS], "active groups should cap at MAX_GROUPS") + + +def test_normalize_time_zone_keeps_valid_names_and_rejects_junk(): + require(delivery.normalize_time_zone("America/New_York") == "America/New_York", "valid IANA-looking zone should be kept") + for zone in (None, "", "Europe/Paris;drop", "A" * 65, "America New_York", "Etc/UTC