From 3e03d692609daeea3624ee6c95b1202c7abc897b Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 1 Oct 2026 16:36:53 -0400 Subject: [PATCH 01/36] Capture workflow chat delivery off golden on unmodified base Captured on b61729627 (app code identical to b5cc18ee2) before any 6b-1 change. With Use Workflow Results In Chat off and workflow runs on, the P5 note, sidecar, first run write, and answer metadata must stay byte-identical. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../workflow_chat_delivery_off_golden.json | 332 ++++++++++++++++++ .../test_workflow_chat_delivery_off_golden.py | 193 ++++++++++ 2 files changed, 525 insertions(+) create mode 100644 functional_tests/test_support/workflow_chat_delivery_off_golden.json create mode 100644 functional_tests/test_workflow_chat_delivery_off_golden.py diff --git a/functional_tests/test_support/workflow_chat_delivery_off_golden.json b/functional_tests/test_support/workflow_chat_delivery_off_golden.json new file mode 100644 index 000000000..32ce05667 --- /dev/null +++ b/functional_tests/test_support/workflow_chat_delivery_off_golden.json @@ -0,0 +1,332 @@ +{ + "answer_and_run": { + "first_write": [ + { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + ], + "message": { + "_etag": "<_etag>", + "agent_citations": [], + "augmented": false, + "content": "Your priorities.\n\nSaved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "conversation_id": "conversation-1", + "generated_artifacts": [], + "hybrid_citations": [], + "id": "assistant_orchestration_e4eec6448d10579e9dc3ef181f1393b1db184560", + "metadata": { + "orchestration": { + "attempt_index": 1, + "failure": null, + "failures": [], + "finalization_status": "saved", + "message_saved": true, + "outcome": "completed", + "outputs": [], + "plan_summary": { + "capabilities_used": [ + "compose", + "workflow_run" + ], + "intent_summary": "", + "plan_id": "plan-1", + "run_id": "run-1", + "status": "completed", + "step_count": 2, + "turn_id": "turn-1" + }, + "recovery": { + "eligible": false, + "expected_version": "", + "message": "This request is already complete.", + "reason_code": "already_completed", + "requires_confirmation": false, + "retry_step_ids": [], + "reused_step_ids": [ + "answer", + "run_digest" + ], + "source_run_id": "run-1" + }, + "retry_of_run_id": null, + "run_id": "run-1", + "status": "completed", + "turn_id": "turn-1" + }, + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "token_usage": { + "completion_tokens": 3, + "prompt_tokens": 7, + "total_tokens": 10 + } + }, + "model_deployment_name": "gpt-4o", + "model_provider": "aoai", + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "role": "assistant", + "timestamp": "", + "web_search_citations": [] + }, + "notes": { + "one": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "stopped": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.\n\nStopping this plan doesn't stop a workflow it already started. Cancel the run in Workflows if you need to.", + "two": "Saved workflows:\n- Started `Weekly digest`.\n- Started `Other digest`.\n\nFollow each run's progress and results in that workflow's run history in Workflows. Results also appear wherever each workflow already sends them, such as its conversation or alerts." + }, + "queued": [ + { + "actor_user_id": "owner", + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "request_id": "023264b5-a9ca-5c85-a03e-f351f8b7f984", + "trigger_source": "chat_orchestration" + } + ], + "sidecar": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "handle": "workflow-weekly-digest-702b6c", + "name": "Weekly digest", + "orchestration_run_id": "run-1", + "reason": null, + "requested_by": "owner", + "run_id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "status": "queued", + "step_id": "run_digest", + "version": 1, + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001" + }, + "step_summary": "Started the saved workflow.", + "stored_run": { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + }, + "run_only": { + "first_write": [ + { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + ], + "message": { + "_etag": "<_etag>", + "agent_citations": [], + "augmented": false, + "content": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "conversation_id": "conversation-1", + "generated_artifacts": [], + "hybrid_citations": [], + "id": "assistant_orchestration_e4eec6448d10579e9dc3ef181f1393b1db184560", + "metadata": { + "orchestration": { + "attempt_index": 1, + "failure": null, + "failures": [], + "finalization_status": "saved", + "message_saved": true, + "outcome": "completed", + "outputs": [], + "plan_summary": { + "capabilities_used": [ + "workflow_run" + ], + "intent_summary": "", + "plan_id": "plan-1", + "run_id": "run-1", + "status": "completed", + "step_count": 1, + "turn_id": "turn-1" + }, + "recovery": { + "eligible": false, + "expected_version": "", + "message": "This request is already complete.", + "reason_code": "already_completed", + "requires_confirmation": false, + "retry_step_ids": [], + "reused_step_ids": [ + "run_digest" + ], + "source_run_id": "run-1" + }, + "retry_of_run_id": null, + "run_id": "run-1", + "status": "completed", + "turn_id": "turn-1" + }, + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "token_usage": { + "completion_tokens": 0, + "prompt_tokens": 0, + "total_tokens": 0 + } + }, + "model_deployment_name": "gpt-4o", + "model_provider": "aoai", + "reasoning_adjustments": [], + "reasoning_effort": null, + "reasoning_mode": "model_default", + "requested_reasoning_effort": null, + "role": "assistant", + "timestamp": "", + "web_search_citations": [] + }, + "notes": { + "one": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.", + "stopped": "Saved workflows:\n- Started `Weekly digest`.\n\nFollow the run's progress and results in the workflow's run history in Workflows. Results also appear wherever the workflow already sends them, such as its conversation or alerts.\n\nStopping this plan doesn't stop a workflow it already started. Cancel the run in Workflows if you need to.", + "two": "Saved workflows:\n- Started `Weekly digest`.\n- Started `Other digest`.\n\nFollow each run's progress and results in that workflow's run history in Workflows. Results also appear wherever each workflow already sends them, such as its conversation or alerts." + }, + "queued": [ + { + "actor_user_id": "owner", + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "request_id": "023264b5-a9ca-5c85-a03e-f351f8b7f984", + "trigger_source": "chat_orchestration" + } + ], + "sidecar": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "handle": "workflow-weekly-digest-702b6c", + "name": "Weekly digest", + "orchestration_run_id": "run-1", + "reason": null, + "requested_by": "owner", + "run_id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "status": "queued", + "step_id": "run_digest", + "version": 1, + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001" + }, + "step_summary": "Started the saved workflow.", + "stored_run": { + "chat_invocation": { + "attempt_root_run_id": "run-1", + "conversation_id": "conversation-1", + "orchestration_run_id": "run-1", + "requested_at": "", + "requested_by": "owner", + "source": "chat_orchestration", + "step_id": "run_digest", + "user_message_id": "user-turn-1", + "version": 1 + }, + "completed_at": null, + "definition_revision": "46952c927c274ab52531aef33c39968964a4d3f59e24760654aac131ebf651cf", + "definition_version": 1, + "durable_execution": true, + "group_id": null, + "id": "88417c5f-aa7b-5964-a897-cba2212302ec", + "started_at": "2026-09-29T12:19:00+00:00", + "status": "queued", + "success": false, + "trigger_source": "chat_orchestration", + "triggered_by": "owner", + "user_id": "owner", + "workflow_id": "6d2e8c20-3f4a-4b5c-8d6e-000000000001", + "workflow_name": "Weekly digest", + "workspace_type": "personal" + } + } +} diff --git a/functional_tests/test_workflow_chat_delivery_off_golden.py b/functional_tests/test_workflow_chat_delivery_off_golden.py new file mode 100644 index 000000000..54143d3b4 --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_off_golden.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_off_golden.py +""" +Functional test for the workflow results to chat setting-off golden. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures that, with ``enable_chat_workflow_results`` off (the default) and chat +orchestration workflow runs on, starting a saved workflow from chat is byte-identical to the +release before results were posted back to chat. It compares, against a committed golden +fixture captured on the unmodified base (0.261.216, b6172962777f, before any delivery change): + +- the step sidecar ``result['workflow_run']`` and the step summary; +- the options the step passes to the durable queue; +- the body of the run document's first write and the stored run document; +- the reply's note, from the real sidecar, for one run, two runs and a stopped plan; +- the answer message's content and metadata. + +Never regenerate the fixture to make this test pass. After merging an upstream change to how a +plan starts workflows, regenerate it from the upstream commit itself (without this feature): + + python functional_tests/test_workflow_chat_delivery_off_golden.py --write-golden +""" + +import json +import os +import sys +from copy import deepcopy +from difflib import unified_diff +from pathlib import Path + +import pytest + +from test_orchestration_harness_routes import modules # noqa: F401 +from test_orchestration_workflow_run_adapter import ( # noqa: F401 + RUN_ID, + _signed_in, + world, + wr, +) +from test_orchestration_workflow_run_answer import _plan_harness +from test_orchestration_workflow_run_capability import _handle, _run, planning # noqa: F401 +from test_orchestration_workflow_run_planning_context import OWNER, wf # noqa: F401 +from test_support.orchestration_harness_execution import compose_step, decoded_frames, input_binding +from test_support.versioning import assert_app_version_at_least + + +GOLDEN = Path(__file__).resolve().parent / "test_support" / "workflow_chat_delivery_off_golden.json" +WRITE_ENV = "SIMPLECHAT_WRITE_WORKFLOW_CHAT_DELIVERY_OFF_GOLDEN" +RESULTS_SETTING = "enable_chat_workflow_results" +# The 6a setting is off by default, so it is compared both absent and explicitly off. +SETTING_VARIANTS = {"absent": None, "off": False} +FLOWS = ("answer_and_run", "run_only") +# Values that change on every run. Each is replaced by a placeholder that names it, so the rest of +# the document is compared byte for byte. +VOLATILE_KEYS = frozenset({ + "requested_at", "timestamp", "created_at", "updated_at", "last_updated", "_ts", "_etag", + "thread_id", "previous_thread_id", "expected_version", +}) + + +def test_version_includes_the_workflow_chat_delivery_off_golden(): + assert_app_version_at_least("0.261.218") + + +def _pretty(value): + return json.dumps(value, indent=1, ensure_ascii=False, sort_keys=True) + + +def _diff(expected, actual, label): + return "\n".join(unified_diff( + expected.splitlines(), actual.splitlines(), fromfile=f"golden {label}", tofile=f"actual {label}", + lineterm="", + )) + + +def _normalized(value): + if isinstance(value, dict): + return { + key: (f"<{key}>" if key in VOLATILE_KEYS and item not in (None, "") else _normalized(item)) + for key, item in value.items() + } + if isinstance(value, list): + return [_normalized(item) for item in value] + return value + + +def _environment(monkeypatch, planning, flow): + run_step = _run(_handle(planning, "Weekly digest")) + if flow == "answer_and_run": + return _plan_harness( + monkeypatch, planning, [compose_step("answer"), run_step], replies=["Your priorities."], + final_response=input_binding("answer"), + ) + return _plan_harness(monkeypatch, planning, [run_step]) + + +def _capture(world, wr, planning, monkeypatch, flow, results_setting): + created = [] + real_create = world.runs.create_item + + def create_item(body, **kwargs): + # The body of the run document's first write, before storage adds anything. + created.append(deepcopy(body)) + return real_create(body, **kwargs) + + monkeypatch.setattr(world.runs, "create_item", create_item) + env = _environment(monkeypatch, planning, flow) + if results_setting is not None: + env.settings[RESULTS_SETTING] = results_setting + else: + env.settings.pop(RESULTS_SETTING, None) + execution = _signed_in(env.prepare()) + decoded_frames(execution.execute(emit=lambda frame: None)) + messages = env.assistant_messages() + assert len(messages) == 1, messages + step = env.steps.read_item("run-1:run_digest", "run-1") + sidecar = step["workflow_run"] + stored = world.runs.get(OWNER, RUN_ID) + plan_one = {"steps": [{"step_id": "run_digest", "capability_id": step["capability_id"], "enabled": True}]} + plan_two = {"steps": [ + *plan_one["steps"], {"step_id": "run_other", "capability_id": step["capability_id"], "enabled": True}, + ]} + record = {"step_id": "run_digest", "capability_id": step["capability_id"], "status": step["status"], + "workflow_run": deepcopy(sidecar)} + other = {**deepcopy(record), "step_id": "run_other", + "workflow_run": {**deepcopy(sidecar), "step_id": "run_other", "name": "Other digest"}} + return _normalized({ + "sidecar": sidecar, + "step_summary": step.get("summary"), + "queued": world.queued, + "first_write": created, + "stored_run": {key: value for key, value in (stored or {}).items() if not key.startswith("_")}, + "notes": { + "one": wr.workflow_run_note(plan_one, [record]), + "two": wr.workflow_run_note(plan_two, [record, other]), + "stopped": wr.workflow_run_note(plan_one, [record], stopped=True), + }, + "message": messages[0], + }) + + +def _load_golden(): + return json.loads(GOLDEN.read_text(encoding="utf-8")) + + +def _write_section(flow, captured): + golden = _load_golden() if GOLDEN.exists() else {} + golden[flow] = captured + GOLDEN.write_text(_pretty(golden) + "\n", encoding="utf-8") + + +@pytest.mark.parametrize("variant", sorted(SETTING_VARIANTS)) +@pytest.mark.parametrize("flow", FLOWS) +def test_starting_a_workflow_with_results_in_chat_off_is_byte_identical_to_the_base( + world, wr, planning, monkeypatch, flow, variant, +): + captured = _capture(world, wr, planning, monkeypatch, flow, SETTING_VARIANTS[variant]) + if os.environ.get(WRITE_ENV) == "1": + if variant == "absent": + _write_section(flow, captured) + return + expected = _load_golden()[flow] + for part in sorted(expected): + expected_text, actual_text = _pretty(expected[part]), _pretty(captured.get(part)) + assert actual_text == expected_text, _diff(expected_text, actual_text, f"{flow} {part}") + assert sorted(captured) == sorted(expected) + + +def test_the_fixture_covers_the_cases_that_matter(): + golden = _load_golden() + assert sorted(golden) == sorted(FLOWS) + for flow in FLOWS: + case = golden[flow] + assert case["sidecar"]["status"] == "queued" and case["sidecar"]["run_id"] == RUN_ID + assert "chat_delivery" not in case["sidecar"] + assert len(case["queued"]) == 1 and "chat_delivery" not in case["queued"][0] + assert len(case["first_write"]) == 1 and "chat_delivery" not in case["first_write"][0] + assert "chat_invocation" in case["first_write"][0] + assert "chat_delivery" not in case["stored_run"] + assert "post" not in case["notes"]["one"].lower() + assert case["message"]["content"].endswith(case["notes"]["one"]) + assert "workflow_delivery" not in json.dumps(case["message"]) + assert golden["answer_and_run"]["message"]["content"].startswith("Your priorities.\n\n") + + +if __name__ == "__main__": + if "--write-golden" in sys.argv: + os.environ[WRITE_ENV] = "1" + exit_code = pytest.main([__file__, "-q", "-k", "byte_identical_to_the_base and absent"]) + print(f"Wrote {GOLDEN}" if exit_code == 0 else "The golden was not written.") + sys.exit(exit_code) + sys.exit(pytest.main([__file__, "-q"])) From 06f554dbe04a4b63f5e5ea7e143f0292c7146bb8 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 1 Oct 2026 21:49:42 -0400 Subject: [PATCH 02/36] Add workflow chat delivery contract module (6b-1) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_workflow_chat_delivery.py | 915 ++++++++++++++++++ 1 file changed, 915 insertions(+) create mode 100644 application/single_app/functions_workflow_chat_delivery.py diff --git a/application/single_app/functions_workflow_chat_delivery.py b/application/single_app/functions_workflow_chat_delivery.py new file mode 100644 index 000000000..34392e449 --- /dev/null +++ b/application/single_app/functions_workflow_chat_delivery.py @@ -0,0 +1,915 @@ +# functions_workflow_chat_delivery.py +"""Pure chat delivery contract for chat-started workflow runs. + +Version: 0.261.218 +Implemented in: 0.261.218 + +This module owns the pure contract for posting a chat-started workflow run's result back into the +chat. It performs no storage, model calls, or result-text reads. Inline copies of ``_fingerprint``, +``clean_catalog_text``, ``quoted_workflow_name``, state sets, and model identity fields are pinned +by parity tests because importing their sources would create import cycles or pull azure.cosmos or +config into this module. +""" + +import hashlib +import json +import logging +import re +import threading +import uuid +from collections import deque +from collections.abc import Mapping +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from urllib.parse import urlencode + +from functions_appinsights import log_event, workflow_log_context +from functions_m365_workflow_binding import M365_WAITING_STATES + + +CHAT_DELIVERY_VERSION = 1 +CHAT_DELIVERY_KEY = 'chat_delivery' +DELIVERY_METADATA_KEY = 'workflow_delivery' +DELIVERY_MESSAGE_ID_PREFIX = 'assistant_workflow_delivery_' +NOTIFICATION_TYPE = 'workflow_chat_delivery' +WORKFLOW_SCOPE = 'personal' +CHAT_TRIGGER_SOURCE = 'chat_orchestration' +LOG_PREFIX = '[WORKFLOW_CHAT_DELIVERY]' +STATUS_LOG_PREFIX = '[WORKFLOW_CHAT_DELIVERY_STATUS]' + +STATUS_PENDING = 'pending' +STATUS_READY = 'ready' +STATUS_DELIVERING = 'delivering' +STATUS_DELIVERED = 'delivered' +STATUS_UNDELIVERABLE = 'undeliverable' +STATUS_EXPIRED = 'expired' +STATUSES = frozenset({ + STATUS_PENDING, + STATUS_READY, + STATUS_DELIVERING, + STATUS_DELIVERED, + STATUS_UNDELIVERABLE, + STATUS_EXPIRED, +}) +OPEN_STATUSES = frozenset({STATUS_PENDING, STATUS_READY, STATUS_DELIVERING}) +FINAL_STATUSES = frozenset({STATUS_DELIVERED, STATUS_UNDELIVERABLE, STATUS_EXPIRED}) +REOPENABLE_STATUSES = frozenset({STATUS_DELIVERED, STATUS_UNDELIVERABLE}) + +KIND_RESULT = 'result' +KIND_FAILED = 'failed' +KIND_CANCELLED = 'cancelled' +KIND_STATUS = 'status' +KIND_CONTENT_BLOCKED = 'content_blocked' +KIND_ANALYSIS = 'analysis' +KIND_EXPIRED = 'expired' +KIND_SKIPPED = 'skipped' +RESULT_CONTEXT_KINDS = frozenset({KIND_RESULT, KIND_ANALYSIS}) + +PHASE_CLAIMED = 'claimed' +PHASE_PUBLISHING = 'publishing' +PHASE_UNREAD_MARKED = 'unread_marked' +PHASE_MESSAGE_CREATED = 'message_created' +PHASE_NOTIFIED = 'notified' +PHASES = (PHASE_CLAIMED, PHASE_PUBLISHING, PHASE_UNREAD_MARKED, PHASE_MESSAGE_CREATED, PHASE_NOTIFIED) + +NOTICE_CHAT_RESPONSE = 'chat_response' +NOTICE_UNDELIVERABLE = 'undeliverable' +NOTICE_EXPIRED = 'expired' +NOTICE_NONE = 'none' +NOTICE_KINDS = frozenset({NOTICE_CHAT_RESPONSE, NOTICE_UNDELIVERABLE, NOTICE_EXPIRED, NOTICE_NONE}) + +REASON_CHAT_UNAVAILABLE = 'chat_unavailable' +REASON_ACCESS_LOST = 'access_lost' +REASON_WORKFLOW_DELETED = 'workflow_deleted' +REASON_RUNTIME_MISSING = 'runtime_missing' +REASON_DELIVERY_FAILED = 'delivery_failed' +REASON_EXPIRED_BEFORE_DELIVERY = 'expired_before_delivery' +REASON_CONTENT_BLOCKED = 'content_blocked' +REASON_RESULTS_OFF = 'results_off' +REASON_RESULT_UNAVAILABLE = 'result_unavailable' +REASON_DEADLINE_EXCEEDED = 'deadline_exceeded' +SILENT_REASONS = frozenset({REASON_WORKFLOW_DELETED, REASON_RUNTIME_MISSING}) + +RUNTIME_TERMINAL_STATES = frozenset({ + 'completed', + 'completed_partial', + 'failed', + 'invalid', + 'incomplete', + 'cancelled', + 'skipped', +}) +RESULT_RUN_STATES = frozenset({'completed', 'completed_partial'}) +FAILED_RUN_STATES = frozenset({'failed', 'invalid', 'incomplete'}) +RUNTIME_RESUMABLE_STATES = frozenset({'failed', 'invalid', 'incomplete'}) +RUNTIME_WAITING_STATES = frozenset({'waiting_approval', 'waiting_output', 'waiting_recovery', 'paused'}) | M365_WAITING_STATES +RUNTIME_ACTIVE_STATES = frozenset({'queued', 'running', 'cancelling'}) +RUN_DOCUMENT_TERMINAL_STATUSES = RUNTIME_TERMINAL_STATES | frozenset({'canceled'}) + +KIND_BY_TERMINAL_STATE = { + 'completed': KIND_RESULT, + 'completed_partial': KIND_RESULT, + 'failed': KIND_FAILED, + 'invalid': KIND_FAILED, + 'incomplete': KIND_FAILED, + 'cancelled': KIND_CANCELLED, + 'skipped': KIND_SKIPPED, +} +MODEL_IDENTITY_FIELDS = ('model_deployment', 'model_id', 'model_endpoint_id', 'model_provider') + +DELIVERY_GRACE_SECONDS = 86400 +DEFAULT_DEADLINE_SECONDS = 86400 +MAX_DEADLINE_SECONDS = 86400 +LEASE_SECONDS = 300 +SWEEP_INTERVAL_SECONDS = 30 +SWEEP_LOCK_NAME = 'workflow_chat_delivery_sweep' +SWEEP_LOCK_SECONDS = 120 +SWEEP_TOP = 25 +SWEEP_MAX_RUNS = 8 +SWEEP_MAX_SECONDS = 90 +SWEEP_TS_GRACE_SECONDS = 120 +BACKOFF_SECONDS = (30, 60, 120, 240, 480, 900, 900, 900) +MAX_ATTEMPTS = 8 +DEFER_RETRY_SECONDS = 30 +DEFER_MAX_SECONDS = 900 +STREAM_META_FRESH_SECONDS = 90 +RECENT_USER_MESSAGE_SECONDS = 600 + +HINT_QUEUE_MAX = 256 +HISTORY_MAX = 5 +MAX_ROLES = 20 +MAX_ROLE_LENGTH = 64 +MAX_GROUPS = 10 +MAX_GROUP_ID_LENGTH = 64 +MAX_MODEL_FIELD_LENGTH = 200 +MAX_REASONING_EFFORT_LENGTH = 32 +NAME_MAX_LENGTH = 80 + +COMPOSE_TEMPERATURE = 0.3 +COMPOSE_MAX_TOKENS = 1500 +REPLY_MAX_CHARS = 12000 +TRIMMED_FALLBACK_CHARS = 2000 +EXCERPT_BUDGET_BYTES = 48 * 1024 +TRIMMED_FALLBACK_INTRO = "I couldn't summarize the result, so here's the start of it:" +FALLBACK_QUESTION = 'Summarize what this workflow run produced.' +WORKFLOW_RUN_DELIVERY_FOLLOW_UP = ( + "I'll post the results here when the run finishes. You can also follow it in the workflow's " + "run history in Workflows." +) +WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY = ( + "I'll post each run's results here when it finishes. You can also follow them in each " + "workflow's run history in Workflows." +) +UNDELIVERABLE_NOTICE_MESSAGE = "The chat that started this run can't show it anymore. Open the run in Workflows to see the details." +EXPIRED_NOTICE_MESSAGE = "The run didn't finish in time to post to the chat. Open it in Workflows to see where it stands." +_DEFAULT_WORKFLOW_NAME = 'Workflow' +_TEXT_NOISE = re.compile('[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff]') +_TIME_ZONE_RE = re.compile(r'[A-Za-z0-9_+\-/]{1,64}') + +_HINT_LOCK = threading.Lock() +_HINT_EVENT = threading.Event() +_HINT_QUEUE = deque() +_HINT_KEYS = set() + + +def _fingerprint(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode('utf-8') + ).hexdigest() + + +def clean_catalog_text(value, limit): + """Return user-authored text as one bounded, printable line, or '' for anything else.""" + if not isinstance(value, str): + return '' + text = ' '.join(_TEXT_NOISE.sub(' ', value).split()) + if len(text) <= limit: + return text + return text[:limit - 1].rstrip() + '\u2026' + + +def quoted_workflow_name(value): + """A saved name as inline code that cannot format the reply, or None when nothing is left of it.""" + text = clean_catalog_text(value, NAME_MAX_LENGTH) + if not text.replace('`', '').strip(): + return None + return '`' + text.replace('`', "'") + '`' + + +def display_workflow_name(value): + return quoted_workflow_name(value) or f'`{_DEFAULT_WORKFLOW_NAME}`' + + +def utc_now(): + return datetime.now(timezone.utc) + + +def format_delivery_timestamp(dt): + if not isinstance(dt, datetime): + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + else: + dt = dt.astimezone(timezone.utc) + return dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +def parse_delivery_timestamp(value): + if isinstance(value, datetime): + dt = value + elif isinstance(value, str): + text = value.strip() + if not text or len(text) > 64: + return None + if text.endswith('Z'): + text = text[:-1] + '+00:00' + try: + dt = datetime.fromisoformat(text) + except (TypeError, ValueError): + return None + else: + return None + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def workflow_delivery_message_id(run_id, conversation_id, generation): + if not isinstance(run_id, str) or not run_id.strip(): + raise ValueError('run_id is required') + if not isinstance(conversation_id, str) or not conversation_id.strip(): + raise ValueError('conversation_id is required') + if type(generation) is not int or generation < 0: + raise ValueError('generation must be a non-negative integer') + return DELIVERY_MESSAGE_ID_PREFIX + _fingerprint([run_id, conversation_id, generation])[:40] + + +def delivery_thread_id(message_id): + if not isinstance(message_id, str) or not message_id.strip(): + raise ValueError('message_id is required') + return str(uuid.uuid5(uuid.NAMESPACE_URL, message_id)) + + +def is_workflow_delivery_message(message): + if not isinstance(message, Mapping): + return False + message_id = message.get('id') + if isinstance(message_id, str) and message_id.startswith(DELIVERY_MESSAGE_ID_PREFIX): + return True + metadata = message.get('metadata') + return isinstance(metadata, Mapping) and isinstance(metadata.get(DELIVERY_METADATA_KEY), Mapping) + + +def normalize_requester_roles(roles): + if not isinstance(roles, (list, tuple)): + return [] + normalized = [] + seen = set() + for role in roles: + if not isinstance(role, str) or not role.strip() or len(role) > MAX_ROLE_LENGTH or role in seen: + continue + normalized.append(role) + seen.add(role) + if len(normalized) >= MAX_ROLES: + break + return normalized + + +def normalize_model_selection(seeds, active_group_ids=None): + if not isinstance(seeds, Mapping): + seeds = {} + model = {} + raw_model = seeds.get('model') + if isinstance(raw_model, Mapping): + invalid = False + for field in MODEL_IDENTITY_FIELDS: + value = raw_model.get(field) + if value is None or value == '': + continue + if not isinstance(value, str): + invalid = True + break + text = value.strip() + if not text or len(text) > MAX_MODEL_FIELD_LENGTH: + invalid = True + break + model[field] = text + if invalid: + model = {} + reasoning_effort = seeds.get('reasoning_effort') + if isinstance(reasoning_effort, str): + reasoning_effort = reasoning_effort.strip() + if len(reasoning_effort) > MAX_REASONING_EFFORT_LENGTH: + reasoning_effort = '' + else: + reasoning_effort = '' + groups_source = seeds.get('active_group_ids') + if not isinstance(groups_source, (list, tuple)) or not groups_source: + groups_source = active_group_ids + active_groups = [] + seen_groups = set() + if isinstance(groups_source, (list, tuple)): + for group_id in groups_source: + if not isinstance(group_id, str): + continue + text = group_id.strip() + if not text or len(text) > MAX_GROUP_ID_LENGTH or text in seen_groups: + continue + active_groups.append(text) + seen_groups.add(text) + if len(active_groups) >= MAX_GROUPS: + break + return {'model': model, 'reasoning_effort': reasoning_effort, 'active_group_ids': active_groups} + + +def normalize_time_zone(value): + if isinstance(value, str) and _TIME_ZONE_RE.fullmatch(value): + return value + return 'UTC' + + +def build_chat_delivery_seed(*, time_zone, model_selection, requester_roles, now=None): + created_at = format_delivery_timestamp(now if isinstance(now, datetime) else utc_now()) + return { + 'version': CHAT_DELIVERY_VERSION, + 'status': STATUS_PENDING, + 'generation': None, + 'expires_at': None, + 'time_zone': normalize_time_zone(time_zone), + 'model_selection': normalize_model_selection(model_selection), + 'requester_roles': normalize_requester_roles(requester_roles), + 'lease_id': None, + 'lease_expires_at': None, + 'attempts': 0, + 'next_attempt_at': None, + 'first_deferred_at': None, + 'phase': None, + 'message_id': None, + 'planned_at': None, + 'kind': None, + 'notice_kind': None, + 'outcome_reason': None, + 'run_status': None, + 'delivered_at': None, + 'created_at': created_at, + 'updated_at': created_at, + 'history': [], + } + + +def _lower_text(value): + if not isinstance(value, str): + return None + text = value.strip().lower() + return text or None + + +def control_summary(control): + if not isinstance(control, Mapping): + return { + 'exists': False, + 'deleted': False, + 'version': None, + 'state': None, + 'phase': None, + 'gate_reason_code': None, + 'deadline_at': None, + 'deadline_seconds': None, + 'schema_version': None, + } + limits = control.get('limits') if isinstance(control.get('limits'), Mapping) else {} + gate = control.get('gate') if isinstance(control.get('gate'), Mapping) else {} + version = control.get('version') + schema_version = control.get('schema_version') + return { + 'exists': True, + 'deleted': bool(control.get('deleted')), + 'version': version if type(version) is int else None, + 'state': _lower_text(control.get('state') if control.get('state') is not None else control.get('control_state')), + 'phase': _lower_text(control.get('phase')), + 'gate_reason_code': _lower_text(gate.get('reason_code')), + 'deadline_at': control.get('deadline_at') if control.get('deadline_at') is not None else limits.get('deadline_at'), + 'deadline_seconds': control.get('deadline_seconds') if control.get('deadline_seconds') is not None else limits.get('deadline_seconds'), + 'schema_version': schema_version if type(schema_version) is int else None, + } + + +def compute_expires_at(*, deadline_at=None, deadline_seconds=None, base=None): + parsed_deadline = parse_delivery_timestamp(deadline_at) + if parsed_deadline: + return format_delivery_timestamp(parsed_deadline + timedelta(seconds=DELIVERY_GRACE_SECONDS)) + parsed_base = parse_delivery_timestamp(base) or utc_now() + seconds = DEFAULT_DEADLINE_SECONDS + if type(deadline_seconds) is int and deadline_seconds > 0: + seconds = min(deadline_seconds, MAX_DEADLINE_SECONDS) + return format_delivery_timestamp(parsed_base + timedelta(seconds=seconds + DELIVERY_GRACE_SECONDS)) + + +def finalize_chat_delivery_seed(seed, control): + if not isinstance(seed, Mapping): + return None + record = deepcopy(dict(seed)) + summary = control_summary(control) + record['expires_at'] = compute_expires_at( + deadline_at=summary.get('deadline_at'), + deadline_seconds=summary.get('deadline_seconds'), + base=record.get('created_at'), + ) + return record + + +def chat_delivery_applies(run, conversation_id): + if not isinstance(run, Mapping) or not isinstance(conversation_id, str) or not conversation_id: + return False + delivery = run.get(CHAT_DELIVERY_KEY) + invocation = run.get('chat_invocation') + return ( + isinstance(delivery, Mapping) + and type(delivery.get('version')) is int + and delivery.get('version') == CHAT_DELIVERY_VERSION + and isinstance(invocation, Mapping) + and invocation.get('conversation_id') == conversation_id + ) + + +def needs_guarded_run_save(run): + if not isinstance(run, Mapping): + return False + return CHAT_DELIVERY_KEY in run or (run.get('trigger_source') == CHAT_TRIGGER_SOURCE and 'chat_invocation' not in run) + + +def merge_stored_run_fields(incoming, stored): + incoming = incoming if isinstance(incoming, Mapping) else {} + stored = stored if isinstance(stored, Mapping) else {} + cosmos_keys = {'_rid', '_self', '_etag', '_attachments', '_ts'} + merged = {key: deepcopy(value) for key, value in incoming.items() if key not in cosmos_keys} + if CHAT_DELIVERY_KEY in stored: + merged[CHAT_DELIVERY_KEY] = deepcopy(stored.get(CHAT_DELIVERY_KEY)) + else: + merged.pop(CHAT_DELIVERY_KEY, None) + if 'chat_invocation' not in merged and 'chat_invocation' in stored: + merged['chat_invocation'] = deepcopy(stored.get('chat_invocation')) + stored_status = _lower_text(stored.get('status')) + has_cancel_request = bool(stored.get('cancellation_requested_at')) or stored_status in {'cancelling', 'cancelled', 'canceled'} + if has_cancel_request: + for field in ('cancellation_requested_at', 'cancellation_requested_by'): + if stored.get(field): + merged[field] = deepcopy(stored.get(field)) + incoming_status = _lower_text(merged.get('status')) + if stored_status in {'cancelled', 'canceled'}: + merged['status'] = 'cancelled' + elif incoming_status not in {'cancelled', 'canceled'}: + merged['status'] = 'cancelling' + return merged + + +def _mark_closed(record, reason, timestamp): + updated = deepcopy(dict(record)) + updated.update({ + 'status': STATUS_UNDELIVERABLE, + 'outcome_reason': reason, + 'notice_kind': NOTICE_NONE, + 'lease_id': None, + 'lease_expires_at': None, + 'next_attempt_at': None, + 'updated_at': timestamp, + }) + return updated + + +def _generation_from_summary(summary): + version = summary.get('version') if isinstance(summary, Mapping) else None + if type(version) is int and version >= 0: + return version + return 0 + + +def _ready_fields(summary, kind): + state = summary.get('state') if isinstance(summary, Mapping) else None + return { + 'status': STATUS_READY, + 'generation': _generation_from_summary(summary), + 'kind': kind, + 'run_status': state, + } + + +def _history_entry(record): + return { + 'generation': record.get('generation'), + 'status': record.get('status'), + 'kind': record.get('kind'), + 'outcome_reason': record.get('outcome_reason'), + 'message_id': record.get('message_id'), + 'delivered_at': record.get('delivered_at'), + 'notice_kind': record.get('notice_kind'), + } + + +def _changed_record(original, candidate): + original_without_updated = {key: value for key, value in original.items() if key != 'updated_at'} + candidate_without_updated = {key: value for key, value in candidate.items() if key != 'updated_at'} + return original_without_updated != candidate_without_updated + + +def reconcile_chat_delivery(record, summary, now=None): + timestamp_dt = now if isinstance(now, datetime) else utc_now() + timestamp = format_delivery_timestamp(timestamp_dt) + if not isinstance(record, Mapping): + return deepcopy(record), False, False + current = deepcopy(dict(record)) + if current.get('version') != CHAT_DELIVERY_VERSION or current.get('status') not in STATUSES: + return current, False, False + status = current.get('status') + if status in {STATUS_DELIVERING, STATUS_EXPIRED}: + return current, False, False + if not isinstance(summary, Mapping) or not summary.get('exists'): + if status in {STATUS_PENDING, STATUS_READY}: + updated = _mark_closed(current, REASON_RUNTIME_MISSING, timestamp) + return updated, True, False + return current, False, False + if summary.get('deleted'): + if status in {STATUS_PENDING, STATUS_READY}: + updated = _mark_closed(current, REASON_WORKFLOW_DELETED, timestamp) + return updated, True, False + return current, False, False + reopened = False + if status in REOPENABLE_STATUSES and type(current.get('generation')) is int: + version = summary.get('version') + if type(version) is int and version > current.get('generation'): + history = list(current.get('history') if isinstance(current.get('history'), list) else []) + history.append(_history_entry(current)) + current.update({ + 'status': STATUS_PENDING, + 'generation': None, + 'kind': None, + 'notice_kind': None, + 'outcome_reason': None, + 'run_status': None, + 'delivered_at': None, + 'message_id': None, + 'planned_at': None, + 'phase': None, + 'lease_id': None, + 'lease_expires_at': None, + 'next_attempt_at': None, + 'first_deferred_at': None, + 'attempts': 0, + 'expires_at': compute_expires_at( + deadline_at=summary.get('deadline_at'), + deadline_seconds=summary.get('deadline_seconds'), + base=timestamp_dt, + ), + 'history': history[-HISTORY_MAX:], + }) + status = STATUS_PENDING + reopened = True + if status in {STATUS_PENDING, STATUS_READY}: + state = summary.get('state') + candidate = deepcopy(current) + expires_at = parse_delivery_timestamp(candidate.get('expires_at')) + if state in RUNTIME_TERMINAL_STATES: + candidate.update(_ready_fields(summary, KIND_BY_TERMINAL_STATE[state])) + elif ( + state == 'paused' + and REASON_DEADLINE_EXCEEDED in {summary.get('gate_reason_code'), summary.get('phase')} + ) or (state not in RUNTIME_TERMINAL_STATES and expires_at and timestamp_dt > expires_at): + candidate.update(_ready_fields(summary, KIND_EXPIRED)) + elif status == STATUS_READY: + candidate.update({'status': STATUS_PENDING, 'generation': None, 'kind': None, 'run_status': None}) + if reopened or _changed_record(current, candidate): + candidate['updated_at'] = timestamp + return candidate, True, candidate.get('status') == STATUS_READY + return current, False, current.get('status') == STATUS_READY + return current, False, False + + +def phase_at_least(phase, wanted): + if phase not in PHASES or wanted not in PHASES: + return False + return PHASES.index(phase) >= PHASES.index(wanted) + + +def delivery_label(workflow_name, asked_when): + label = f'Results from {display_workflow_name(workflow_name)}' + if isinstance(asked_when, str) and asked_when: + return f'{label} ยท you asked on {asked_when}' + return label + + +def assemble_delivery_content(label, body, disclosure=''): + parts = [] + if isinstance(label, str) and label: + parts.append(label) + body_text = body.strip() if isinstance(body, str) else '' + if body_text: + parts.append(body_text) + if isinstance(disclosure, str) and disclosure: + parts.append(disclosure) + return '\n\n'.join(parts) + + +def failure_reason_text(code): + reasons = { + 'failed': 'the run stopped before it finished', + 'invalid': "the workflow couldn't run as defined", + 'incomplete': "some of its steps didn't finish", + 'skipped': 'no new or changed files were detected', + REASON_DEADLINE_EXCEEDED: 'it reached its time limit', + 'execution_budget_exceeded': 'it reached its execution limit', + 'repeat_iteration_limit': 'it reached its repeat limit', + 'm365_authorization': 'Microsoft 365 access needs to be reconnected', + 'authorization': 'your workflow access must be restored', + } + return reasons.get(code, reasons['failed']) + + +def delivery_note_text(kind, workflow_name, *, reason_code=None): + name = display_workflow_name(workflow_name) + if kind == KIND_FAILED: + return f'{name} failed: {failure_reason_text(reason_code)}.' + if kind == KIND_CANCELLED: + return f'{name} was cancelled.' + if kind == KIND_STATUS: + return f'{name} finished. Open the run to see its results.' + if kind == KIND_CONTENT_BLOCKED: + return f"{name} finished, but its results can't be shown here. Open the run to see them." + if kind == KIND_ANALYSIS: + return f'{name} finished with a saved analysis. Ask a follow-up question about it here.' + if kind == KIND_SKIPPED: + return f"{name} didn't run: no new or changed files were detected." + raise ValueError('unsupported delivery note kind') + + +def notice_workflow_name(name): + text = clean_catalog_text(name, NAME_MAX_LENGTH).replace('"', "'") + if not text: + return 'your workflow' + return f'"{text}"' + + +def _sentence_start(text): + if text.startswith('your workflow'): + return 'Your workflow' + text[len('your workflow'):] + return text + + +def notice_title(kind, workflow_name): + q = notice_workflow_name(workflow_name) + if kind in {KIND_RESULT, KIND_ANALYSIS, KIND_STATUS, KIND_CONTENT_BLOCKED}: + return _sentence_start(f'Results from {q} are ready') + if kind == KIND_FAILED: + return _sentence_start(f"{q} didn't finish") + if kind == KIND_CANCELLED: + return _sentence_start(f'{q} was cancelled') + if kind == KIND_SKIPPED: + return _sentence_start(f"{q} didn't run") + if kind == KIND_EXPIRED: + return _sentence_start(f"{q} didn't finish in time to post to chat") + raise ValueError('unsupported notice kind') + + +def delivered_notice_preview(workflow_name): + return _sentence_start(f'Results from {notice_workflow_name(workflow_name)} are in your chat') + + +def delivery_notification_key(run_id, generation): + return f'workflow-chat-delivery:{run_id}:{generation}' + + +def undeliverable_notification_key(run_id, generation): + return f'workflow-chat-delivery-notice:{run_id}:{generation}' + + +def expired_notification_key(run_id): + return f'workflow-chat-delivery-notice:{run_id}:expired' + + +def token_usage_idempotency_key(message_id, attempt): + return f'workflow_result_delivery:{message_id}:{attempt}' + + +def workflow_run_notice_link(workflow_id, run_id): + return '/workflow-activity?' + urlencode({'workflowId': workflow_id, 'runId': run_id, 'scope': WORKFLOW_SCOPE}) + + +def notice_metadata(workflow_id, run_id, delivery_status): + return {'workflow_id': workflow_id, 'run_id': run_id, 'workflow_scope': WORKFLOW_SCOPE, 'delivery_status': delivery_status} + + +def build_delivery_metadata(*, kind, workflow_id, run_id, generation, run_status, orchestration_run_id, step_id, requested_at): + return { + 'version': CHAT_DELIVERY_VERSION, + 'kind': kind, + 'workflow_id': workflow_id, + 'workflow_scope': WORKFLOW_SCOPE, + 'run_id': run_id, + 'generation': generation, + 'run_status': run_status, + 'orchestration_run_id': orchestration_run_id, + 'step_id': step_id, + 'requested_at': requested_at, + } + + +def next_backoff_seconds(attempts): + if type(attempts) is not int: + return BACKOFF_SECONDS[0] + index = min(max(attempts, 1), len(BACKOFF_SECONDS)) - 1 + return BACKOFF_SECONDS[index] + + +def signal_workflow_chat_delivery(user_id, run_id): + try: + if not isinstance(user_id, str) or not user_id.strip() or not isinstance(run_id, str) or not run_id.strip(): + return False + key = (user_id, run_id) + with _HINT_LOCK: + if key in _HINT_KEYS: + _HINT_EVENT.set() + return True + if len(_HINT_QUEUE) >= HINT_QUEUE_MAX: + return False + _HINT_QUEUE.append(key) + _HINT_KEYS.add(key) + _HINT_EVENT.set() + return True + except Exception: + return False + + +def drain_workflow_chat_delivery_hints(limit=None): + with _HINT_LOCK: + if type(limit) is not int or limit < 0: + limit = len(_HINT_QUEUE) + drained = [] + while _HINT_QUEUE and len(drained) < limit: + key = _HINT_QUEUE.popleft() + _HINT_KEYS.discard(key) + drained.append(key) + if not _HINT_QUEUE: + _HINT_EVENT.clear() + return drained + + +def wait_for_workflow_chat_delivery_hint(timeout): + return _HINT_EVENT.wait(timeout) + + +def clear_workflow_chat_delivery_hints(): + with _HINT_LOCK: + _HINT_QUEUE.clear() + _HINT_KEYS.clear() + _HINT_EVENT.clear() + + +def delivery_log(message, *, level=logging.INFO, status_route=False, run_id=None, conversation_id=None, **fields): + try: + prefix = STATUS_LOG_PREFIX if status_route else LOG_PREFIX + stage = 'workflow_chat_delivery_status' if status_route else 'workflow_chat_delivery' + extra = {'stage': stage, **workflow_log_context(conversation_id=conversation_id, run_id=run_id)} + extra.update({key: value for key, value in fields.items() if value is not None}) + log_event(f'{prefix} {message}', extra=extra, level=level) + except Exception: + return + + +__all__ = ( + 'CHAT_DELIVERY_VERSION', + 'CHAT_DELIVERY_KEY', + 'DELIVERY_METADATA_KEY', + 'DELIVERY_MESSAGE_ID_PREFIX', + 'NOTIFICATION_TYPE', + 'WORKFLOW_SCOPE', + 'CHAT_TRIGGER_SOURCE', + 'LOG_PREFIX', + 'STATUS_LOG_PREFIX', + 'STATUS_PENDING', + 'STATUS_READY', + 'STATUS_DELIVERING', + 'STATUS_DELIVERED', + 'STATUS_UNDELIVERABLE', + 'STATUS_EXPIRED', + 'STATUSES', + 'OPEN_STATUSES', + 'FINAL_STATUSES', + 'REOPENABLE_STATUSES', + 'KIND_RESULT', + 'KIND_FAILED', + 'KIND_CANCELLED', + 'KIND_STATUS', + 'KIND_CONTENT_BLOCKED', + 'KIND_ANALYSIS', + 'KIND_EXPIRED', + 'KIND_SKIPPED', + 'RESULT_CONTEXT_KINDS', + 'PHASE_CLAIMED', + 'PHASE_PUBLISHING', + 'PHASE_UNREAD_MARKED', + 'PHASE_MESSAGE_CREATED', + 'PHASE_NOTIFIED', + 'PHASES', + 'NOTICE_CHAT_RESPONSE', + 'NOTICE_UNDELIVERABLE', + 'NOTICE_EXPIRED', + 'NOTICE_NONE', + 'NOTICE_KINDS', + 'REASON_CHAT_UNAVAILABLE', + 'REASON_ACCESS_LOST', + 'REASON_WORKFLOW_DELETED', + 'REASON_RUNTIME_MISSING', + 'REASON_DELIVERY_FAILED', + 'REASON_EXPIRED_BEFORE_DELIVERY', + 'REASON_CONTENT_BLOCKED', + 'REASON_RESULTS_OFF', + 'REASON_RESULT_UNAVAILABLE', + 'REASON_DEADLINE_EXCEEDED', + 'SILENT_REASONS', + 'RUNTIME_TERMINAL_STATES', + 'RESULT_RUN_STATES', + 'FAILED_RUN_STATES', + 'RUNTIME_RESUMABLE_STATES', + 'RUNTIME_WAITING_STATES', + 'RUNTIME_ACTIVE_STATES', + 'RUN_DOCUMENT_TERMINAL_STATUSES', + 'KIND_BY_TERMINAL_STATE', + 'MODEL_IDENTITY_FIELDS', + 'DELIVERY_GRACE_SECONDS', + 'DEFAULT_DEADLINE_SECONDS', + 'MAX_DEADLINE_SECONDS', + 'LEASE_SECONDS', + 'SWEEP_INTERVAL_SECONDS', + 'SWEEP_LOCK_NAME', + 'SWEEP_LOCK_SECONDS', + 'SWEEP_TOP', + 'SWEEP_MAX_RUNS', + 'SWEEP_MAX_SECONDS', + 'SWEEP_TS_GRACE_SECONDS', + 'BACKOFF_SECONDS', + 'MAX_ATTEMPTS', + 'DEFER_RETRY_SECONDS', + 'DEFER_MAX_SECONDS', + 'STREAM_META_FRESH_SECONDS', + 'RECENT_USER_MESSAGE_SECONDS', + 'HINT_QUEUE_MAX', + 'HISTORY_MAX', + 'MAX_ROLES', + 'MAX_ROLE_LENGTH', + 'MAX_GROUPS', + 'MAX_GROUP_ID_LENGTH', + 'MAX_MODEL_FIELD_LENGTH', + 'MAX_REASONING_EFFORT_LENGTH', + 'NAME_MAX_LENGTH', + 'COMPOSE_TEMPERATURE', + 'COMPOSE_MAX_TOKENS', + 'REPLY_MAX_CHARS', + 'TRIMMED_FALLBACK_CHARS', + 'EXCERPT_BUDGET_BYTES', + 'TRIMMED_FALLBACK_INTRO', + 'FALLBACK_QUESTION', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY', + 'UNDELIVERABLE_NOTICE_MESSAGE', + 'EXPIRED_NOTICE_MESSAGE', + 'clean_catalog_text', + 'quoted_workflow_name', + 'display_workflow_name', + 'utc_now', + 'format_delivery_timestamp', + 'parse_delivery_timestamp', + 'workflow_delivery_message_id', + 'delivery_thread_id', + 'is_workflow_delivery_message', + 'normalize_requester_roles', + 'normalize_model_selection', + 'normalize_time_zone', + 'build_chat_delivery_seed', + 'control_summary', + 'compute_expires_at', + 'finalize_chat_delivery_seed', + 'chat_delivery_applies', + 'needs_guarded_run_save', + 'merge_stored_run_fields', + 'reconcile_chat_delivery', + 'phase_at_least', + 'delivery_label', + 'assemble_delivery_content', + 'delivery_note_text', + 'failure_reason_text', + 'notice_workflow_name', + 'notice_title', + 'delivered_notice_preview', + 'delivery_notification_key', + 'undeliverable_notification_key', + 'expired_notification_key', + 'token_usage_idempotency_key', + 'workflow_run_notice_link', + 'notice_metadata', + 'build_delivery_metadata', + 'next_backoff_seconds', + 'signal_workflow_chat_delivery', + 'drain_workflow_chat_delivery_hints', + 'wait_for_workflow_chat_delivery_hint', + 'clear_workflow_chat_delivery_hints', + 'delivery_log', +) From 776baa5d526f4d13f0bdba0f5f4c80692eb8d642 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 1 Oct 2026 22:00:19 -0400 Subject: [PATCH 03/36] Record chat delivery when a chat plan starts a workflow run Seeds the run's chat_delivery record behind Use Workflow Results In Chat, flags the sidecar, and promises the result in the chat only when every started run will be posted back. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_orchestration_workflow_runs.py | 77 +++++++++++++++++-- .../single_app/functions_workflow_runtime.py | 6 +- ...test_orchestration_workflow_run_adapter.py | 2 + 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/application/single_app/functions_orchestration_workflow_runs.py b/application/single_app/functions_orchestration_workflow_runs.py index 7d2dfbfb7..c63c35391 100644 --- a/application/single_app/functions_orchestration_workflow_runs.py +++ b/application/single_app/functions_orchestration_workflow_runs.py @@ -73,6 +73,14 @@ workflow_run_ready, workflow_run_settings_gate, ) +from functions_workflow_chat_delivery import ( + WORKFLOW_RUN_DELIVERY_FOLLOW_UP, + WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY, + build_chat_delivery_seed, + chat_delivery_applies, + normalize_model_selection, + normalize_requester_roles, +) WORKFLOW_RUN_OUTPUT = 'run' @@ -200,8 +208,9 @@ } _DEFAULT_WORKFLOW_NAME = 'Workflow' -# The reply's note about the saved workflows a plan started. It says where a run's results appear -# and never promises them in the chat, which only a later delivery step can do. +# The reply's note about the saved workflows a plan started. It says where a run's results appear. +# It promises them in the chat only when every run it started recorded a chat delivery, because only +# then does the server post each run's result back into this conversation when the run finishes. WORKFLOW_RUN_NOTE_HEADING = 'Saved workflows:' WORKFLOW_RUN_FOLLOW_UP = ( "Follow the run's progress and results in the workflow's run history in Workflows. Results also " @@ -585,6 +594,40 @@ def _started_status(run): return WORKFLOW_RUN_STATUS_QUEUED +def _chat_delivery_seed(settings, context, planning): + """Return the record that has the run's result posted back to this chat, or None. + + The cheap settings check comes first, so a deployment with Use Workflow Results In Chat off + never reaches the role-aware gate. A failure here only means the run is not delivered: it + never fails the start. + """ + if not isinstance(settings, dict) or not settings.get('enable_chat_workflow_results'): + return None + try: + # Settings initialize application storage, so they are imported only once a gate is reached. + from functions_settings import is_chat_workflow_results_enabled_for_user + + user_roles = getattr(context, 'user_roles', None) + if not is_chat_workflow_results_enabled_for_user(settings, user_roles=user_roles): + return None + time_zone = getattr(context, 'time_zone', None) + if not time_zone and isinstance(planning, dict): + time_zone = planning.get('time_zone') + return build_chat_delivery_seed( + time_zone=time_zone, + model_selection=normalize_model_selection( + getattr(context, 'seeds', None), getattr(context, 'active_group_ids', None), + ), + requester_roles=normalize_requester_roles(user_roles), + ) + except Exception as exc: + _log( + 'Chat delivery was not recorded for a started workflow run.', logging.WARNING, + run_id=getattr(context, 'run_id', None), error_type=type(exc).__name__, + ) + return None + + def _start(step, context, *, settings, user_id, recheck, requested_at): """Start or link the step's workflow run. Returns an outcome; raises only to fail the step. @@ -651,6 +694,7 @@ def unavailable(reason, workflow_id=None): return unavailable(RULE_NOT_DURABLE, workflow_id) if workflow.get('status') in M365_ACTIVE_STATES: return unavailable(REASON_WAITING_FOR_MICROSOFT_365, workflow_id) + chat_delivery = _chat_delivery_seed(settings, context, planning) recheck() runtime_unavailable, runtime_conflict = _runtime_errors() @@ -671,6 +715,7 @@ def unavailable(reason, workflow_id=None): 'requested_by': user_id, 'requested_at': requested_at, }, + **({'chat_delivery': chat_delivery} if chat_delivery is not None else {}), ) except runtime_conflict as exc: code = getattr(exc, 'code', None) @@ -684,11 +729,19 @@ def unavailable(reason, workflow_id=None): except AzureError as exc: raise _WorkflowRunStepFailure(FAILURE_RUNTIME_UNAVAILABLE) from exc run = queued.get('run') if isinstance(queued, dict) else None - return _outcome(_started_status(run), name=name, workflow_id=workflow_id, run_id=run_id, queued=True) + outcome = _outcome(_started_status(run), name=name, workflow_id=workflow_id, run_id=run_id, queued=True) + # Only a run this call started, carrying this chat's delivery record, is posted back here. + if ( + chat_delivery is not None + and outcome['status'] in (WORKFLOW_RUN_STATUS_QUEUED, WORKFLOW_RUN_STATUS_RUNNING) + and chat_delivery_applies(run, context.conversation_id) + ): + outcome['chat_delivery'] = True + return outcome def _sidecar(step, context, *, user_id, producer_run_id, outcome): - return { + sidecar = { 'version': WORKFLOW_RUN_VERSION, 'step_id': step['step_id'], 'orchestration_run_id': producer_run_id, @@ -702,6 +755,10 @@ def _sidecar(step, context, *, user_id, producer_run_id, outcome): 'status': outcome.get('status'), 'reason': outcome.get('reason'), } + # Set only by a start that recorded this chat's delivery, never for a linked or rebuilt run. + if outcome.get('chat_delivery') is True: + sidecar['chat_delivery'] = True + return sidecar def _failure(exc): @@ -879,7 +936,7 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): record['step_id']: record for record in (execution_steps if isinstance(execution_steps, list) else ()) if isinstance(record, dict) and isinstance(record.get('step_id'), str) } - lines, started = [], 0 + lines, started, delivered = [], 0, 0 for step in steps: if not isinstance(step, dict) or step.get('capability_id') != CAPABILITY_WORKFLOW_RUN: continue @@ -896,6 +953,8 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): continue if sidecar['status'] in WORKFLOW_RUN_STARTED_STATUSES: started += 1 + if sidecar.get('chat_delivery') is True: + delivered += 1 lines.append(_step_line(sidecar)) notes = plan.get('workflow_run_notes') if isinstance(plan.get('workflow_run_notes'), list) else [] for note in notes: @@ -909,7 +968,11 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): return '' parts = ['\n'.join([WORKFLOW_RUN_NOTE_HEADING, *lines])] if started: - parts.append(WORKFLOW_RUN_FOLLOW_UP_MANY if started > 1 else WORKFLOW_RUN_FOLLOW_UP) + # The chat promises the result only when every run started here will be posted back to it. + if delivered == started: + parts.append(WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY if started > 1 else WORKFLOW_RUN_DELIVERY_FOLLOW_UP) + else: + parts.append(WORKFLOW_RUN_FOLLOW_UP_MANY if started > 1 else WORKFLOW_RUN_FOLLOW_UP) if stopped: parts.append(WORKFLOW_RUN_STOPPED) return '\n\n'.join(parts) @@ -919,6 +982,8 @@ def workflow_run_note(plan, execution_steps, *, stopped=False): 'FAILURE_RUNTIME_UNAVAILABLE', 'NO_WORKFLOW_STARTED', 'REASON_INVALID', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP', + 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY', 'WORKFLOW_RUN_FOLLOW_UP', 'WORKFLOW_RUN_FOLLOW_UP_MANY', 'WORKFLOW_RUN_NOTE_HEADING', diff --git a/application/single_app/functions_workflow_runtime.py b/application/single_app/functions_workflow_runtime.py index 74b6c21b1..dff0d05bb 100644 --- a/application/single_app/functions_workflow_runtime.py +++ b/application/single_app/functions_workflow_runtime.py @@ -16,6 +16,7 @@ from functions_appinsights import log_event from functions_m365_workflow_binding import M365_WAITING_STATES +from functions_workflow_chat_delivery import CHAT_DELIVERY_KEY, finalize_chat_delivery_seed from functions_workflow_definitions import ( WORKFLOW_DEFINITION_FIELDS, validate_workflow_publication_completion, workflow_definition_revision, ) @@ -149,7 +150,7 @@ def workflow_run_id_for_request(workflow, request_id): def queue_durable_workflow_run( workflow, *, actor_user_id, trigger_source="manual", request_id=None, invocation_metadata=None, - chat_invocation=None, + chat_invocation=None, chat_delivery=None, ): services = _services(workflow) settings = services["settings"]() @@ -227,6 +228,9 @@ def queue_durable_workflow_run( if chat_invocation is not None: # Which chat orchestration step started this run, for the requester's own run history. run["chat_invocation"] = deepcopy(chat_invocation) + if chat_delivery is not None: + # The record that has this run's result posted back to that chat when the run finishes. + run[CHAT_DELIVERY_KEY] = finalize_chat_delivery_seed(chat_delivery, control) try: saved = services["runs"].read_item(item=run_id, partition_key=services["partition"]) except CosmosResourceNotFoundError: diff --git a/functional_tests/test_orchestration_workflow_run_adapter.py b/functional_tests/test_orchestration_workflow_run_adapter.py index 025db5264..1c88e492d 100644 --- a/functional_tests/test_orchestration_workflow_run_adapter.py +++ b/functional_tests/test_orchestration_workflow_run_adapter.py @@ -1150,6 +1150,8 @@ def test_the_run_module_writes_only_through_the_queue_and_never_imports_flask(): "RUNTIME_TERMINAL_STATES", "queue_durable_workflow_run", "workflow_run_id_for_request", }, "functions_workflow_runtime_store": {"RuntimeUnavailable", "WorkflowRuntimeConflict"}, + # Use Workflow Results In Chat's role-aware gate, read only when a run is about to start. + "functions_settings": {"is_chat_workflow_results_enabled_for_user"}, } calls = _calls_by_function(tree) writes = {"create_item", "upsert_item", "replace_item", "delete_item", "patch_item", "execute_item_batch"} From ceb15642655fe972ca5fe98759d65843b54ebadf Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 1 Oct 2026 22:21:18 -0400 Subject: [PATCH 04/36] Guard chat-started run saves and reconcile chat delivery in the runtime projection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_personal_workflows.py | 46 +++++++++++++++++++ .../single_app/functions_workflow_runtime.py | 35 +++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/application/single_app/functions_personal_workflows.py b/application/single_app/functions_personal_workflows.py index 589ca9b2a..1b9aa57ea 100644 --- a/application/single_app/functions_personal_workflows.py +++ b/application/single_app/functions_personal_workflows.py @@ -8,6 +8,7 @@ import uuid from datetime import datetime, timedelta, timezone +from azure.core import MatchConditions from azure.cosmos import exceptions from config import ( @@ -40,6 +41,7 @@ from functions_settings import get_settings, get_user_settings, normalize_model_endpoints from functions_workflow_alerts import normalize_workflow_alert_settings from functions_workflow_alert_safety import sanitize_workflow_alert_record +from functions_workflow_chat_delivery import delivery_log, merge_stored_run_fields, needs_guarded_run_save from functions_workflow_result_store import delete_workflow_run_results from functions_workflow_bindings import authorize_workflow_reference from functions_workflow_definition_store import ( @@ -86,6 +88,8 @@ WORKFLOW_TASK_NAME_MAX_LENGTH = 120 WORKFLOW_TASK_RUNNER_TYPES = {'inherit', 'agent', 'model'} WORKFLOW_CONVERSATION_ACCESS_ERROR = 'Workflow conversation not found or access denied.' +# A chat-started run's guarded save re-reads and retries at most this many times on a conflict. +_GUARDED_RUN_SAVE_ATTEMPTS = 8 def _utc_now(): return datetime.now(timezone.utc) @@ -1274,10 +1278,52 @@ def save_personal_workflow_run(user_id, run_record): run_record = run_record if isinstance(run_record, dict) else {} run_record['user_id'] = user_id run_record.setdefault('id', str(uuid.uuid4())) + if needs_guarded_run_save(run_record): + return _strip_cosmos_metadata(_save_chat_started_workflow_run(user_id, run_record)) result = cosmos_personal_workflow_runs_container.upsert_item(body=run_record) return _strip_cosmos_metadata(result) +def _save_chat_started_workflow_run(user_id, run_record): + """Save a run a chat plan started without undoing what another writer stored meanwhile. + + The stored chat delivery record, chat invocation and cancellation request win over this copy, + and each replace lands only on the stored document it was merged from. A save that cannot be + guarded falls back to an upsert of the best merge it has, which the delivery's own message id, + notification key and unread rules keep from posting or announcing a result twice. + """ + merged = run_record + for _attempt in range(_GUARDED_RUN_SAVE_ATTEMPTS): + try: + stored = cosmos_personal_workflow_runs_container.read_item( + item=run_record['id'], partition_key=user_id, + ) + except exceptions.CosmosResourceNotFoundError: + return cosmos_personal_workflow_runs_container.upsert_item(body=run_record) + except Exception as exc: + delivery_log( + 'A chat-started workflow run was saved without its guard.', level=logging.WARNING, + run_id=run_record.get('id'), reason='read_failed', error_type=type(exc).__name__, + ) + return cosmos_personal_workflow_runs_container.upsert_item(body=merged) + merged = merge_stored_run_fields(run_record, stored) + try: + return cosmos_personal_workflow_runs_container.replace_item( + item=run_record['id'], body=merged, etag=stored.get('_etag'), + match_condition=MatchConditions.IfNotModified, + ) + except exceptions.CosmosResourceNotFoundError: + return cosmos_personal_workflow_runs_container.upsert_item(body=merged) + except exceptions.CosmosHttpResponseError as exc: + if exc.status_code != 412: + raise + delivery_log( + 'A chat-started workflow run was saved without its guard.', level=logging.WARNING, + run_id=run_record.get('id'), reason='conflicts_exhausted', + ) + return cosmos_personal_workflow_runs_container.upsert_item(body=merged) + + def save_personal_workflow_run_item(user_id, item_record): """Create or update a per-item workflow run record.""" item_record = item_record if isinstance(item_record, dict) else {} diff --git a/application/single_app/functions_workflow_runtime.py b/application/single_app/functions_workflow_runtime.py index dff0d05bb..7a38920c7 100644 --- a/application/single_app/functions_workflow_runtime.py +++ b/application/single_app/functions_workflow_runtime.py @@ -16,7 +16,10 @@ from functions_appinsights import log_event from functions_m365_workflow_binding import M365_WAITING_STATES -from functions_workflow_chat_delivery import CHAT_DELIVERY_KEY, finalize_chat_delivery_seed +from functions_workflow_chat_delivery import ( + CHAT_DELIVERY_KEY, control_summary, delivery_log, finalize_chat_delivery_seed, reconcile_chat_delivery, + signal_workflow_chat_delivery, +) from functions_workflow_definitions import ( WORKFLOW_DEFINITION_FIELDS, validate_workflow_publication_completion, workflow_definition_revision, ) @@ -251,6 +254,28 @@ def queue_durable_workflow_run( return {"success": True, "run": saved, "workflow": bound, "runtime": workflow_runtime_projection(control)} +def _reconcile_chat_delivery(body, control, run_id): + """Move a chat-started run's delivery record along with its runtime. Never raises. + + Returns True when the record is ready for its result to be posted back to the chat. A failure + keeps the stored record as it was, and the delivery sweep catches it up later. + """ + record = body.get(CHAT_DELIVERY_KEY) + if not isinstance(record, dict): + return False + try: + reconciled, changed, ready = reconcile_chat_delivery(record, control_summary(control)) + except Exception as exc: + delivery_log( + 'A chat delivery record was not reconciled with its run.', level=logging.WARNING, + run_id=run_id, error_type=type(exc).__name__, + ) + return False + if changed: + body[CHAT_DELIVERY_KEY] = reconciled + return bool(ready) + + def _project_runtime_run(services, workflow, run_id, control, *, result=None, attempt=0): latest = workflow_runtime_store(workflow, run_id).read() if latest["version"] > control["version"]: @@ -265,6 +290,11 @@ def _project_runtime_run(services, workflow, run_id, control, *, result=None, at body = {key: value for key, value in existing.items() if not key.startswith("_")} if result: body.update(result.get("run") or {}) + # The stored chat delivery record wins over any copy a result or checkpoint carried. + if CHAT_DELIVERY_KEY in existing: + body[CHAT_DELIVERY_KEY] = deepcopy(existing[CHAT_DELIVERY_KEY]) + else: + body.pop(CHAT_DELIVERY_KEY, None) body.update( durable_execution=True, status=control["state"], runtime=workflow_runtime_projection(control), @@ -276,6 +306,7 @@ def _project_runtime_run(services, workflow, run_id, control, *, result=None, at if int(existing.get("runtime_version") or 0) > control["version"]: return existing body["runtime_version"] = control["version"] + delivery_ready = _reconcile_chat_delivery(body, control, run_id) try: services["runs"].replace_item( item=run_id, body=body, etag=existing["_etag"], match_condition=MatchConditions.IfNotModified, @@ -289,6 +320,8 @@ def _project_runtime_run(services, workflow, run_id, control, *, result=None, at attempt=attempt + 1, ) raise + if delivery_ready: + signal_workflow_chat_delivery(body.get("user_id"), run_id) for _attempt in range(5): current = services["definitions"].read_item(item=workflow["id"], partition_key=services["partition"]) if current.get("deleting"): From 10353bc61bd0c303f8e8bea423643558c9d2aba5 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Thu, 1 Oct 2026 22:32:20 -0400 Subject: [PATCH 05/36] Add the chat delivery notification type, retry-keyed chat notices and a guarded unread mark Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_conversation_unread.py | 110 +++++++++++++++++- .../single_app/functions_notifications.py | 20 +++- 2 files changed, 127 insertions(+), 3 deletions(-) diff --git a/application/single_app/functions_conversation_unread.py b/application/single_app/functions_conversation_unread.py index 583c4080a..a643cbaa2 100644 --- a/application/single_app/functions_conversation_unread.py +++ b/application/single_app/functions_conversation_unread.py @@ -2,7 +2,19 @@ """Helpers for conversation unread assistant-response state.""" -from datetime import datetime +from datetime import datetime, timezone + +from azure.core import MatchConditions +from azure.cosmos import exceptions + + +# Closed outcomes of mark_conversation_unread_guarded. +UNREAD_MARKED = 'marked' +UNREAD_ALREADY_MARKED = 'already_marked' +UNREAD_READ_SINCE = 'read_since' +UNREAD_NOT_FOUND = 'not_found' +UNREAD_UNAVAILABLE = 'unavailable' +UNREAD_CONFLICT = 'conflict' def normalize_conversation_unread_state(conversation_item): @@ -42,3 +54,99 @@ def clear_conversation_unread(conversation_item): normalized_item['last_unread_assistant_message_id'] = None normalized_item['last_unread_assistant_at'] = None return normalized_item + + +def _parse_utc_timestamp(value): + if not isinstance(value, str) or not value.strip(): + return None + try: + parsed = datetime.fromisoformat(value.strip().replace('Z', '+00:00')) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _reached(value, threshold): + """Whether timestamp ``value`` is at or after ``threshold``; False when either is unreadable.""" + parsed_value = _parse_utc_timestamp(value) + parsed_threshold = _parse_utc_timestamp(threshold) + return parsed_value is not None and parsed_threshold is not None and parsed_value >= parsed_threshold + + +def _conversation_unavailable(conversation_item, user_id, require_private): + if not isinstance(conversation_item, dict) or conversation_item.get('user_id') != user_id: + return True + if conversation_item.get('orchestration_deleted') or conversation_item.get('deleted'): + return True + if not require_private: + return False + # Imported here so the plain unread helpers above keep their light import graph. + from functions_orchestration_memory import conversation_is_private + + return not conversation_is_private(conversation_item, user_id) + + +def mark_conversation_unread_guarded( + container, + conversation_id, + user_id, + message_id, + planned_at, + *, + require_private=True, + skip_if_read_since=None, + max_retries=5, +): + """Mark one assistant message unread on a conversation, under an ETag guard. + + Every attempt rereads the conversation and rechecks that it exists, belongs to ``user_id``, + isn't being deleted and, by default, is still private, so a concurrent delete or share wins + over the mark. ``last_updated`` is raised to at least ``planned_at`` so the conversation sorts + with its newest message. + + ``skip_if_read_since`` is the planned time of an earlier attempt. Marking a chat read keeps its + ``last_updated``, so a conversation that already reached that time was either marked and then + read, or moved on without the mark; either way it is left alone. + + Returns ``(outcome, conversation)``. The outcome is one of the ``UNREAD_*`` constants and the + conversation is the stored item when one was read. The caller invalidates any conversation + cache after ``UNREAD_MARKED``. Storage errors other than a missing item or an ETag conflict + propagate. + """ + if not isinstance(message_id, str) or not message_id or not isinstance(planned_at, str) or not planned_at: + raise ValueError('An unread mark needs a message id and a planned time.') + attempts = max(1, int(max_retries) + 1) + conversation_item = None + for _attempt in range(attempts): + try: + conversation_item = container.read_item(item=conversation_id, partition_key=conversation_id) + except exceptions.CosmosResourceNotFoundError: + return UNREAD_NOT_FOUND, None + if _conversation_unavailable(conversation_item, user_id, require_private): + return UNREAD_UNAVAILABLE, conversation_item + if conversation_item.get('last_unread_assistant_message_id') == message_id: + return UNREAD_ALREADY_MARKED, conversation_item + if skip_if_read_since is not None and _reached(conversation_item.get('last_updated'), skip_if_read_since): + return UNREAD_READ_SINCE, conversation_item + + candidate = dict(conversation_item) + if not _reached(candidate.get('last_updated'), planned_at): + candidate['last_updated'] = planned_at + candidate = mark_conversation_unread(candidate, message_id, planned_at) + try: + stored = container.replace_item( + item=conversation_id, + body=candidate, + etag=conversation_item.get('_etag'), + match_condition=MatchConditions.IfNotModified, + ) + except exceptions.CosmosResourceNotFoundError: + return UNREAD_NOT_FOUND, None + except exceptions.CosmosHttpResponseError as exc: + if exc.status_code != 412: + raise + continue + return UNREAD_MARKED, stored if isinstance(stored, dict) else candidate + return UNREAD_CONFLICT, conversation_item diff --git a/application/single_app/functions_notifications.py b/application/single_app/functions_notifications.py index a39a90bb8..d445e6a89 100644 --- a/application/single_app/functions_notifications.py +++ b/application/single_app/functions_notifications.py @@ -34,6 +34,9 @@ KEY_VAULT_SECRET_REMINDER_NOTIFICATION_TYPE = 'key_vault_secret_expiring' M365_APPROVAL_PENDING_NOTIFICATION_TYPE = 'm365_approval_pending' M365_APPROVAL_UPDATED_NOTIFICATION_TYPE = 'm365_approval_updated' +# Pinned by a parity test to functions_workflow_chat_delivery.NOTIFICATION_TYPE, which this module +# does not import so its import graph stays unchanged. +WORKFLOW_CHAT_DELIVERY_NOTIFICATION_TYPE = 'workflow_chat_delivery' MAX_NOTIFICATION_IDEMPOTENCY_KEY_LENGTH = 512 WORKFLOW_ALERT_PRIORITY_CONFIG = { 'info': { @@ -209,6 +212,10 @@ 'icon': 'bi-bell', 'color': 'secondary' }, + WORKFLOW_CHAT_DELIVERY_NOTIFICATION_TYPE: { + 'icon': 'bi-activity', + 'color': 'info' + }, KEY_VAULT_SECRET_REMINDER_NOTIFICATION_TYPE: { 'icon': 'bi-safe', 'color': 'warning' @@ -715,8 +722,15 @@ def create_chat_response_notification( message_id, conversation_title='', response_preview='', + *, + idempotency_key=None, + strict=False, ): - """Create a personal notification when a chat response completes.""" + """Create a personal notification when a chat response completes. + + ``idempotency_key`` and ``strict`` pass through to ``create_notification``, so a server + retry with the same key returns the existing notice rather than creating a second one. + """ normalized_title = str(conversation_title or '').strip() or 'Conversation' normalized_preview = str(response_preview or '').strip() if len(normalized_preview) > 160: @@ -740,7 +754,9 @@ def create_chat_response_notification( metadata={ 'conversation_id': conversation_id, 'message_id': message_id, - } + }, + idempotency_key=idempotency_key, + strict=strict, ) From 81509f3a3d544b348def1d5ae85dcd496d41c49a Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 01:08:09 -0400 Subject: [PATCH 06/36] Add the workflow chat delivery worker (6b-1) Claims a run's chat_delivery record with an ETag-fenced lease, re-checks the chat, the run, the runtime control and both gates, defers mid-stream, composes a bounded fenced reply or a fixed note, marks the chat unread before the deterministic message create, and sends one idempotent notice. Closed chats get no message and one notice. Crashes, outages, deferrals, deletions and durable resumes never post twice or retry forever. Bumps VERSION to 0.261.218. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_workflow_chat_delivery.py | 58 +- ...functions_workflow_chat_delivery_worker.py | 1744 +++++++++++++++++ .../workflow_chat_delivery_fakes.py | 955 +++++++++ .../test_workflow_chat_delivery_worker.py | 1642 ++++++++++++++++ 5 files changed, 4398 insertions(+), 3 deletions(-) create mode 100644 application/single_app/functions_workflow_chat_delivery_worker.py create mode 100644 functional_tests/test_support/workflow_chat_delivery_fakes.py create mode 100644 functional_tests/test_workflow_chat_delivery_worker.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 2584dd8ec..4ef56809c 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -100,7 +100,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.216" +VERSION = "0.261.218" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_workflow_chat_delivery.py b/application/single_app/functions_workflow_chat_delivery.py index 34392e449..265180576 100644 --- a/application/single_app/functions_workflow_chat_delivery.py +++ b/application/single_app/functions_workflow_chat_delivery.py @@ -477,6 +477,52 @@ def _mark_closed(record, reason, timestamp): return updated +def _mark_delivered_without_notice(record, timestamp): + updated = deepcopy(dict(record)) + updated.update({ + 'status': STATUS_DELIVERED, + 'notice_kind': NOTICE_NONE, + 'delivered_at': updated.get('delivered_at') or timestamp, + 'lease_id': None, + 'lease_expires_at': None, + 'next_attempt_at': None, + 'updated_at': timestamp, + }) + return updated + + +def _close_for_runtime(record, reason, timestamp): + # A message that already exists stays delivered; only an unposted generation closes. + if phase_at_least(record.get('phase'), PHASE_MESSAGE_CREATED): + return _mark_delivered_without_notice(record, timestamp) + return _mark_closed(record, reason, timestamp) + + +def _abandon_generation(current, candidate): + """Reset the per-generation fields when a ready generation is replaced before it finished.""" + updated = deepcopy(candidate) + if phase_at_least(current.get('phase'), PHASE_MESSAGE_CREATED): + history = list(current.get('history') if isinstance(current.get('history'), list) else []) + entry = _history_entry(current) + entry.update({'status': STATUS_DELIVERED, 'notice_kind': NOTICE_NONE}) + history.append(entry) + updated['history'] = history[-HISTORY_MAX:] + updated.update({ + 'phase': None, + 'planned_at': None, + 'message_id': None, + 'attempts': 0, + 'outcome_reason': None, + 'notice_kind': None, + 'delivered_at': None, + 'next_attempt_at': None, + 'first_deferred_at': None, + 'lease_id': None, + 'lease_expires_at': None, + }) + return updated + + def _generation_from_summary(summary): version = summary.get('version') if isinstance(summary, Mapping) else None if type(version) is int and version >= 0: @@ -525,12 +571,12 @@ def reconcile_chat_delivery(record, summary, now=None): return current, False, False if not isinstance(summary, Mapping) or not summary.get('exists'): if status in {STATUS_PENDING, STATUS_READY}: - updated = _mark_closed(current, REASON_RUNTIME_MISSING, timestamp) + updated = _close_for_runtime(current, REASON_RUNTIME_MISSING, timestamp) return updated, True, False return current, False, False if summary.get('deleted'): if status in {STATUS_PENDING, STATUS_READY}: - updated = _mark_closed(current, REASON_WORKFLOW_DELETED, timestamp) + updated = _close_for_runtime(current, REASON_WORKFLOW_DELETED, timestamp) return updated, True, False return current, False, False reopened = False @@ -577,6 +623,14 @@ def reconcile_chat_delivery(record, summary, now=None): candidate.update(_ready_fields(summary, KIND_EXPIRED)) elif status == STATUS_READY: candidate.update({'status': STATUS_PENDING, 'generation': None, 'kind': None, 'run_status': None}) + if status == STATUS_READY and candidate.get('generation') != current.get('generation'): + # The run moved on (resumed, or resumed and finished again) before this generation was posted. + candidate = _abandon_generation(current, candidate) + if candidate.get('status') == STATUS_READY and ( + status == STATUS_PENDING or candidate.get('generation') != current.get('generation') + ): + # A record that just became deliverable is due now, not after a wait scheduled while it was pending. + candidate.update({'next_attempt_at': None, 'first_deferred_at': None}) if reopened or _changed_record(current, candidate): candidate['updated_at'] = timestamp return candidate, True, candidate.get('status') == STATUS_READY diff --git a/application/single_app/functions_workflow_chat_delivery_worker.py b/application/single_app/functions_workflow_chat_delivery_worker.py new file mode 100644 index 000000000..d19f1552a --- /dev/null +++ b/application/single_app/functions_workflow_chat_delivery_worker.py @@ -0,0 +1,1744 @@ +# functions_workflow_chat_delivery_worker.py +"""Deliver a chat-started workflow run's outcome back into the chat that asked (6b-1). + +A chat-orchestration plan that starts one of the user's saved workflows ends right away, and the +run may finish hours later. This worker posts the run's outcome into the requesting chat once per +delivery generation: + +* a reply the model composes from a bounded, fenced excerpt of the run's result, under a dated + label; +* a fixed note when the run failed or was cancelled, or its result can't be shown here; +* nothing, and one bell notice instead, when the chat can't take it any more. + +The record that drives delivery is the run document's ``chat_delivery`` field. Only ETag-guarded +writes that fence on the claim's lease advance it. The message ID, the unread mark and every +notification are idempotent, so a crash at any point resumes without a second post or notice. + +This module imports only the standard library, azure-core's match conditions, the unread outcome +constants and the pure delivery contract, so importing it never initializes application storage. +``default_workflow_chat_delivery_services`` binds the production dependencies on first use. +""" + +import logging +import re +import time +import uuid +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Callable, Optional + +from azure.core import MatchConditions + +from functions_workflow_chat_delivery import ( + CHAT_DELIVERY_KEY, + COMPOSE_MAX_TOKENS, + COMPOSE_TEMPERATURE, + DEFER_MAX_SECONDS, + DEFER_RETRY_SECONDS, + DELIVERY_METADATA_KEY, + EXCERPT_BUDGET_BYTES, + EXPIRED_NOTICE_MESSAGE, + FALLBACK_QUESTION, + KIND_ANALYSIS, + KIND_CANCELLED, + KIND_CONTENT_BLOCKED, + KIND_EXPIRED, + KIND_FAILED, + KIND_RESULT, + KIND_SKIPPED, + KIND_STATUS, + LEASE_SECONDS, + MAX_ATTEMPTS, + NOTICE_CHAT_RESPONSE, + NOTICE_EXPIRED, + NOTICE_NONE, + NOTICE_UNDELIVERABLE, + NOTIFICATION_TYPE, + PHASE_CLAIMED, + PHASE_MESSAGE_CREATED, + PHASE_NOTIFIED, + PHASE_PUBLISHING, + PHASE_UNREAD_MARKED, + REASON_ACCESS_LOST, + REASON_CHAT_UNAVAILABLE, + REASON_CONTENT_BLOCKED, + REASON_DELIVERY_FAILED, + REASON_EXPIRED_BEFORE_DELIVERY, + REASON_RESULT_UNAVAILABLE, + REASON_RESULTS_OFF, + REASON_WORKFLOW_DELETED, + RECENT_USER_MESSAGE_SECONDS, + REPLY_MAX_CHARS, + RESULT_CONTEXT_KINDS, + RUN_DOCUMENT_TERMINAL_STATUSES, + SILENT_REASONS, + STATUS_DELIVERED, + STATUS_DELIVERING, + STATUS_EXPIRED, + STATUS_PENDING, + STATUS_READY, + STATUS_UNDELIVERABLE, + STREAM_META_FRESH_SECONDS, + SWEEP_LOCK_NAME, + SWEEP_LOCK_SECONDS, + SWEEP_MAX_RUNS, + SWEEP_MAX_SECONDS, + SWEEP_TOP, + SWEEP_TS_GRACE_SECONDS, + TRIMMED_FALLBACK_CHARS, + TRIMMED_FALLBACK_INTRO, + UNDELIVERABLE_NOTICE_MESSAGE, + assemble_delivery_content, + build_delivery_metadata, + control_summary, + delivered_notice_preview, + delivery_label, + delivery_log, + delivery_note_text, + delivery_notification_key, + delivery_thread_id, + expired_notification_key, + format_delivery_timestamp, + next_backoff_seconds, + notice_metadata, + notice_title, + parse_delivery_timestamp, + phase_at_least, + reconcile_chat_delivery, + token_usage_idempotency_key, + undeliverable_notification_key, + utc_now, + workflow_delivery_message_id, + workflow_run_notice_link, +) +from functions_workflow_chat_delivery import _close_for_runtime +from functions_conversation_unread import ( + UNREAD_ALREADY_MARKED, + UNREAD_MARKED, + UNREAD_NOT_FOUND, + UNREAD_READ_SINCE, + UNREAD_UNAVAILABLE, +) + + +QUESTION_MAX_CHARS = 4000 +RECORD_WRITE_RETRIES = 5 +HINT_BATCH = 32 +NEWEST_MESSAGE_QUERY = ( + 'SELECT TOP 1 c.id, c.role, c.timestamp, c.metadata.thread_info AS thread_info ' + 'FROM c WHERE c.conversation_id = @conversation_id ORDER BY c.timestamp DESC' +) + +OUTCOME_DELIVERED = 'delivered' +OUTCOME_UNDELIVERABLE = 'undeliverable' +OUTCOME_CLOSED_SILENTLY = 'closed_silently' +OUTCOME_EXPIRED = 'expired' +OUTCOME_DEFERRED = 'deferred' +OUTCOME_RETRY_SCHEDULED = 'retry_scheduled' +OUTCOME_PENDING = 'pending' +OUTCOME_NOT_DUE = 'not_due' +OUTCOME_BUSY = 'busy' +OUTCOME_LOST_LEASE = 'lost_lease' +OUTCOME_NOT_APPLICABLE = 'not_applicable' +OUTCOME_UNAVAILABLE = 'unavailable' + + +class _Transient(Exception): + """A retryable failure. It consumes the claim's attempt and backs off.""" + + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +class _LostLease(Exception): + """Another worker owns the record now, or the run document is gone. Stop without writing.""" + + +class _Defer(Exception): + """The chat may be mid-reply. ``counted`` is True when the deferral consumes the attempt.""" + + def __init__(self, counted): + super().__init__('deferred') + self.counted = counted + + +@dataclass +class WorkflowChatDeliveryServices: + """Every dependency the worker touches, so tests can drive it without Azure.""" + + runs: Any + workflows: Any + conversations: Any + messages: Any + get_settings: Callable + runtime_store_factory: Callable + runtime_conflict_error: type + runtime_unavailable_error: type + read_workflow_result: Callable + result_unavailable_error: type + workflow_result_context: Callable + format_run_time: Callable + format_disclosure: Callable + build_result_messages: Callable + new_fence_nonce: Callable + excerpt_budget_steps: tuple + calculate_budget: Callable + resolve_model: Callable + model_error: type + capture_identity: Callable + extract_text: Callable + check_content: Callable + attach_chat_check: Callable + record_incident: Callable + is_message_retracted: Callable + create_chat_response_notification: Callable + create_notification: Callable + log_token_usage: Callable + bump_cache_version: Callable + mark_unread_guarded: Callable + is_conversation_private: Callable + is_user_workflows_enabled: Callable + is_chat_workflow_results_enabled: Callable + stream_activity_probe: Callable + acquire_lock: Callable + release_lock: Callable + drain_hints: Callable + signal_delivery: Callable + clock: Callable + compose_output_tokens: Callable = None + monotonic: Callable = time.monotonic + + +def _no_lock(_task_name, _lease_seconds): + return None + + +def _ignore_release(_lock_document): + return None + + +def _personal_runtime_store(user_id, workflow_id, run_id): + # A personal store's identity is the workflow ID and owner only, so a tombstoned control stays + # readable after the workflow document itself is gone. + from functions_workflow_runtime_store import workflow_runtime_store + + return workflow_runtime_store({'id': workflow_id, 'user_id': user_id}, run_id) + + +def _default_stream_activity_probe(user_id, conversation_id, now): + """Whether a reply may be streaming into this chat right now (D7 signals 1 and 2).""" + import sys + + chats = sys.modules.get('route_backend_chats') + registry = getattr(chats, 'CHAT_STREAM_REGISTRY', None) if chats is not None else None + if registry is not None: + try: + if registry.get_session(user_id, conversation_id, active_only=True): + return True + except Exception as exc: + delivery_log('Stream registry probe failed.', level=logging.WARNING, error_type=type(exc).__name__) + try: + import app_settings_cache + + getter = getattr(app_settings_cache, 'get_stream_session_meta', None) + meta = getter(f'{user_id}:{conversation_id}') if callable(getter) else None + except Exception as exc: + delivery_log('Stream metadata probe failed.', level=logging.WARNING, error_type=type(exc).__name__) + meta = None + if isinstance(meta, dict) and meta.get('active'): + updated = parse_delivery_timestamp(meta.get('updated_at')) + if updated is not None and (now - updated).total_seconds() < STREAM_META_FRESH_SECONDS: + return True + return False + + +def default_workflow_chat_delivery_services(*, acquire_lock=None, release_lock=None): + """Bind the production dependencies. + + Every import is deferred to this call because each one initializes application storage. The + sweep's distributed lock helpers come from background_tasks, which passes them in. Without them + the sweep never runs; hint-driven deliveries still do, because a claim is an ETag write. + """ + import config + import functions_orchestration_models as orchestration_models + from agent_execution_context import capture_execution_identity + from functions_activity_logging import log_token_usage + from functions_chat_content_checks import attach_chat_check, check_chat_content + from functions_chat_content_review import record_chat_content_incident, reply_is_retracted + from functions_conversation_cache import bump_conversation_cache_version + from functions_conversation_unread import mark_conversation_unread_guarded + from functions_notifications import create_chat_response_notification, create_notification + from functions_orchestration_memory import conversation_is_private + from functions_settings import ( + get_settings, + is_chat_workflow_results_enabled_for_user, + is_user_workflows_enabled_for_user, + ) + from functions_workflow_chat_delivery import ( + drain_workflow_chat_delivery_hints, + signal_workflow_chat_delivery, + ) + from functions_workflow_context import calculate_workflow_context_budget + from functions_workflow_result_followup import ( + EXCERPT_BUDGET_STEPS, + build_workflow_result_messages, + new_fence_nonce, + ) + from functions_workflow_result_reader import ( + WorkflowResultUnavailable, + format_workflow_result_disclosure, + format_workflow_run_time, + read_workflow_result, + workflow_result_context, + ) + from functions_workflow_runtime_store import RuntimeUnavailable, WorkflowRuntimeConflict + from model_endpoint_clients import ModelEndpointBehavior, extract_chat_completion_response_text + + def compose_output_tokens(model): + # Mirrors OrchestrationModel.create_completion: reasoning tokens share the completion budget. + behavior = ModelEndpointBehavior(model.provider, getattr(model, 'behavior_name', None) or model.deployment) + if behavior.is_openai_reasoning_model: + return max(COMPOSE_MAX_TOKENS, orchestration_models.REASONING_COMPLETION_BUDGET) + return COMPOSE_MAX_TOKENS + + return WorkflowChatDeliveryServices( + runs=config.cosmos_personal_workflow_runs_container, + workflows=config.cosmos_personal_workflows_container, + conversations=config.cosmos_conversations_container, + messages=config.cosmos_messages_container, + get_settings=get_settings, + runtime_store_factory=_personal_runtime_store, + runtime_conflict_error=WorkflowRuntimeConflict, + runtime_unavailable_error=RuntimeUnavailable, + read_workflow_result=read_workflow_result, + result_unavailable_error=WorkflowResultUnavailable, + workflow_result_context=workflow_result_context, + format_run_time=format_workflow_run_time, + format_disclosure=format_workflow_result_disclosure, + build_result_messages=build_workflow_result_messages, + new_fence_nonce=new_fence_nonce, + excerpt_budget_steps=tuple(EXCERPT_BUDGET_STEPS), + calculate_budget=calculate_workflow_context_budget, + resolve_model=orchestration_models.resolve_orchestration_model, + model_error=orchestration_models.OrchestrationModelError, + capture_identity=capture_execution_identity, + extract_text=extract_chat_completion_response_text, + check_content=check_chat_content, + attach_chat_check=attach_chat_check, + record_incident=record_chat_content_incident, + is_message_retracted=reply_is_retracted, + create_chat_response_notification=create_chat_response_notification, + create_notification=create_notification, + log_token_usage=log_token_usage, + bump_cache_version=bump_conversation_cache_version, + mark_unread_guarded=mark_conversation_unread_guarded, + is_conversation_private=conversation_is_private, + is_user_workflows_enabled=is_user_workflows_enabled_for_user, + is_chat_workflow_results_enabled=is_chat_workflow_results_enabled_for_user, + stream_activity_probe=_default_stream_activity_probe, + acquire_lock=acquire_lock or _no_lock, + release_lock=release_lock or _ignore_release, + drain_hints=drain_workflow_chat_delivery_hints, + signal_delivery=signal_workflow_chat_delivery, + clock=utc_now, + compose_output_tokens=compose_output_tokens, + ) + + +def _status_code(exc): + code = getattr(exc, 'status_code', None) + return code if type(code) is int else None + + +def _message_timestamp(moment): + # Naive ISO UTC with microseconds: the format both conversation loaders already sort. + return format_delivery_timestamp(moment)[:-1] + + +def _now(services): + return parse_delivery_timestamp(services.clock()) or utc_now() + + +def _later(moment, seconds): + return format_delivery_timestamp(moment + timedelta(seconds=seconds)) + + +def _count(value): + return value if type(value) is int and value >= 0 else 0 + + +class _Delivery: + """One claimed generation: the run document as last written and the lease that fences it.""" + + def __init__(self, services, user_id, run, lease_id, *, exhausted=False): + self.services = services + self.user_id = user_id + self.run = run + self.lease_id = lease_id + self.exhausted = exhausted + self.workflow = None + self.settings = None + self.conversation = None + + @property + def run_id(self): + return self.run.get('id') + + @property + def workflow_id(self): + return self.run.get('workflow_id') + + @property + def record(self): + value = self.run.get(CHAT_DELIVERY_KEY) + return dict(value) if isinstance(value, dict) else {} + + @property + def invocation(self): + value = self.run.get('chat_invocation') + return value if isinstance(value, dict) else {} + + @property + def conversation_id(self): + return self.invocation.get('conversation_id') + + @property + def generation(self): + return self.record.get('generation') + + def workflow_name(self): + if isinstance(self.workflow, dict) and isinstance(self.workflow.get('name'), str): + return self.workflow.get('name') + return self.run.get('workflow_name') + + def message_id(self): + return workflow_delivery_message_id(self.run_id, self.conversation_id, self.generation) + + def now(self): + return _now(self.services) + + +def _read_run(services, user_id, run_id): + """The run document, or None when it's gone. Other storage errors propagate.""" + try: + run = services.runs.read_item(item=run_id, partition_key=user_id) + except Exception as exc: + if _status_code(exc) == 404: + return None + raise + if not isinstance(run, dict) or run.get('user_id') != user_id: + return None + return run + + +def _read_control_summary(services, user_id, run): + """Summarize the run's runtime control. A control that was never written reads as missing.""" + try: + store = services.runtime_store_factory(user_id, run.get('workflow_id'), run.get('id')) + control = store.read(allow_deleted=True) + except services.runtime_conflict_error as exc: + if getattr(exc, 'code', None) == 'not_found': + return control_summary(None) + raise _Transient('runtime_conflict') from None + except services.runtime_unavailable_error: + raise _Transient('runtime_unavailable') from None + except Exception as exc: + delivery_log('Runtime control read failed.', level=logging.WARNING, run_id=run.get('id'), error_type=type(exc).__name__) + raise _Transient('runtime_unavailable') from None + return control_summary(control) + + +def _replace_run(services, run, record): + """One ETag-guarded replace of the delivery record: the stored run, or None after a 412.""" + body = dict(run) + body[CHAT_DELIVERY_KEY] = record + try: + stored = services.runs.replace_item( + item=run['id'], body=body, etag=run.get('_etag'), match_condition=MatchConditions.IfNotModified, + ) + except Exception as exc: + code = _status_code(exc) + if code == 412: + return None + if code == 404: + raise _LostLease() from None + delivery_log( + 'Delivery record write failed.', level=logging.WARNING, run_id=run.get('id'), + error_type=type(exc).__name__, code=code, + ) + raise _Transient('record_unavailable') from None + return stored if isinstance(stored, dict) else body + + +def _write_claimed(delivery, mutate, *, release=False, summary=None): + """Apply ``mutate(record, now)`` under the lease and the run's ETag; returns the record written. + + Every write renews the lease unless it releases it. A releasing write is reconciled with + ``summary`` (a fresh control summary) so a resume that landed mid-delivery reopens the record. + """ + services = delivery.services + for _attempt in range(RECORD_WRITE_RETRIES + 1): + current = delivery.run.get(CHAT_DELIVERY_KEY) + if ( + not isinstance(current, dict) or current.get('status') != STATUS_DELIVERING + or current.get('lease_id') != delivery.lease_id + ): + raise _LostLease() + now = delivery.now() + record = mutate(dict(current), now) + record['updated_at'] = format_delivery_timestamp(now) + if release: + record.update({'lease_id': None, 'lease_expires_at': None}) + if summary is not None: + record, _changed, _ready = reconcile_chat_delivery(record, summary, now) + else: + record['lease_expires_at'] = _later(now, LEASE_SECONDS) + stored = _replace_run(services, delivery.run, record) + if stored is not None: + delivery.run = stored + return record + fresh = _read_run(services, delivery.user_id, delivery.run_id) + if fresh is None: + raise _LostLease() + delivery.run = fresh + raise _Transient('record_conflict') + + +def _settled_outcome(record, changed): + status = record.get('status') + if status == STATUS_PENDING: + return OUTCOME_PENDING + if status == STATUS_READY: + return OUTCOME_NOT_DUE + if changed and status in {STATUS_DELIVERED, STATUS_UNDELIVERABLE}: + return OUTCOME_CLOSED_SILENTLY + return OUTCOME_NOT_APPLICABLE + + +def _postpone_unclaimed(services, user_id, run, reason): + """An open record whose control can't be read waits out a backoff, so it never holds a sweep slot. + + The cap closes it without a notice: no outcome was ever read, so there's nothing to tell. + """ + for _attempt in range(RECORD_WRITE_RETRIES + 1): + record = run.get(CHAT_DELIVERY_KEY) if isinstance(run, dict) else None + if not isinstance(record, dict) or record.get('status') not in {STATUS_PENDING, STATUS_READY, STATUS_DELIVERING}: + return OUTCOME_UNAVAILABLE + now = _now(services) + stamp = format_delivery_timestamp(now) + attempts = _count(record.get('attempts')) + 1 + if attempts >= MAX_ATTEMPTS: + updated = _close_for_runtime(record, REASON_DELIVERY_FAILED, stamp) + updated['attempts'] = attempts + else: + updated = dict(record) + updated.update({ + 'status': STATUS_READY if record.get('status') == STATUS_DELIVERING else record.get('status'), + 'lease_id': None, + 'lease_expires_at': None, + 'attempts': attempts, + 'next_attempt_at': _later(now, next_backoff_seconds(attempts)), + 'updated_at': stamp, + }) + stored = _replace_run(services, run, updated) + if stored is not None: + delivery_log( + 'Delivery postponed: the runtime control could not be read.', level=logging.WARNING, + run_id=run.get('id'), reason=reason, attempt=attempts, + ) + if updated.get('status') in {STATUS_DELIVERED, STATUS_UNDELIVERABLE}: + return OUTCOME_CLOSED_SILENTLY + return OUTCOME_RETRY_SCHEDULED + run = _read_run(services, user_id, run.get('id')) + return OUTCOME_BUSY + + +def _claim(services, user_id, run_id): + """Reconcile the record with the runtime, then claim it when a delivery is due. + + Returns a ``_Delivery`` for a claim, otherwise an outcome string. + """ + run = _read_run(services, user_id, run_id) + for _attempt in range(RECORD_WRITE_RETRIES + 1): + record = run.get(CHAT_DELIVERY_KEY) if isinstance(run, dict) else None + if not isinstance(record, dict): + return OUTCOME_NOT_APPLICABLE + now = _now(services) + released = False + if record.get('status') == STATUS_DELIVERING: + lease_end = parse_delivery_timestamp(record.get('lease_expires_at')) + if lease_end is not None and lease_end > now: + return OUTCOME_BUSY + # An abandoned claim is released, then reconciled like any ready record. + record = {**record, 'status': STATUS_READY, 'lease_id': None, 'lease_expires_at': None} + released = True + try: + summary = _read_control_summary(services, user_id, run) + except _Transient as exc: + waiting_until = parse_delivery_timestamp(record.get('next_attempt_at')) + if waiting_until is not None and waiting_until > now: + # An early hint never spends an attempt; the due sweep reads the control again. + return OUTCOME_NOT_DUE + return _postpone_unclaimed(services, user_id, run, exc.reason) + updated, changed, ready = reconcile_chat_delivery(record, summary, now) + due = parse_delivery_timestamp(updated.get('next_attempt_at')) + if ready and (due is None or due <= now): + attempts = _count(updated.get('attempts')) + lease_id = uuid.uuid4().hex + claimed = dict(updated) + claimed.update({ + 'status': STATUS_DELIVERING, + 'lease_id': lease_id, + 'lease_expires_at': _later(now, LEASE_SECONDS), + 'phase': updated.get('phase') or PHASE_CLAIMED, + 'attempts': attempts + 1, + 'next_attempt_at': None, + 'updated_at': format_delivery_timestamp(now), + }) + stored = _replace_run(services, run, claimed) + if stored is not None: + return _Delivery(services, user_id, stored, lease_id, exhausted=attempts >= MAX_ATTEMPTS) + elif changed or released: + stored = _replace_run(services, run, updated) + if stored is not None: + if updated.get('status') == STATUS_READY and due is not None: + return OUTCOME_NOT_DUE + return _settled_outcome(updated, changed) + else: + return _settled_outcome(updated, False) + run = _read_run(services, user_id, run_id) + return OUTCOME_BUSY + + +def _read_workflow(delivery): + """The owner's workflow document, or None once it's gone or being deleted.""" + services = delivery.services + try: + workflow = services.workflows.read_item(item=delivery.workflow_id, partition_key=delivery.user_id) + except Exception as exc: + if _status_code(exc) == 404: + return None + delivery_log( + 'Workflow read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('workflow_unavailable') from None + if not isinstance(workflow, dict) or workflow.get('user_id') != delivery.user_id: + return None + if workflow.get('deleting') or workflow.get('status') == 'deleting': + return None + return workflow + + +def _conversation_ok(services, conversation, user_id): + """Whether the chat can still take a delivery: the owner's, not deleted, and private.""" + if not isinstance(conversation, dict) or conversation.get('user_id') != user_id: + return False + # The same markers the guarded unread mark checks, so both decide alike. + if conversation.get('orchestration_deleted') or conversation.get('deleted'): + return False + try: + return services.is_conversation_private(conversation, user_id) is True + except Exception as exc: + delivery_log('Chat privacy check failed.', level=logging.WARNING, error_type=type(exc).__name__) + raise _Transient('privacy_unavailable') from None + + +def _read_conversation(delivery): + """The requesting chat when it can still take the delivery, otherwise None.""" + services = delivery.services + conversation_id = delivery.conversation_id + if not isinstance(conversation_id, str) or not conversation_id: + return None + try: + conversation = services.conversations.read_item(item=conversation_id, partition_key=conversation_id) + except Exception as exc: + if _status_code(exc) == 404: + return None + delivery_log( + 'Chat read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('conversation_unavailable') from None + if not _conversation_ok(services, conversation, delivery.user_id): + return None + delivery.conversation = conversation + return conversation + + +def _newest_message(delivery): + """The chat's newest message row, or None for an empty chat. + + A failed read is never treated as an empty chat: it defers, and the deferral counts (D7). + """ + services = delivery.services + try: + rows = list(services.messages.query_items( + query=NEWEST_MESSAGE_QUERY, + parameters=[{'name': '@conversation_id', 'value': delivery.conversation_id}], + partition_key=delivery.conversation_id, + )) + except Exception as exc: + delivery_log( + 'Newest-message read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Defer(True) from None + newest = rows[0] if rows else None + return newest if isinstance(newest, dict) else None + + +def _message_exists(delivery, message_id): + """Whether the delivery message was already created by an earlier attempt.""" + services = delivery.services + try: + message = services.messages.read_item(item=message_id, partition_key=delivery.conversation_id) + except Exception as exc: + if _status_code(exc) == 404: + return False + delivery_log( + 'Delivery message read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('message_unavailable') from None + return isinstance(message, dict) + + +def _defer_probe(delivery, record, now): + """Raise ``_Defer`` while a reply may be streaming into the chat (D7). + + Fifteen minutes after the first deferral the activity signals are ignored. A failed + newest-message read still defers, and that deferral consumes the attempt. + """ + newest = _newest_message(delivery) + first = parse_delivery_timestamp(record.get('first_deferred_at')) + if first is not None and (now - first).total_seconds() >= DEFER_MAX_SECONDS: + return newest + if delivery.services.stream_activity_probe(delivery.user_id, delivery.conversation_id, now): + raise _Defer(False) + if isinstance(newest, dict) and newest.get('role') == 'user': + sent = parse_delivery_timestamp(newest.get('timestamp')) + if sent is None or (now - sent).total_seconds() < RECENT_USER_MESSAGE_SECONDS: + raise _Defer(False) + return newest + + +class _Undeliverable(Exception): + """Post nothing for this generation. One notice says why, unless the reason is silent.""" + + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +def _mark_unread(delivery): + """Mark the chat unread for M before M exists (D9 step 6); the mark and T are one write. + + A retry after ``publishing`` passes the earlier planned time, so a chat that already reached + it (marked then read, or moved on) is left alone. + """ + services = delivery.services + record = delivery.record + message_id = delivery.message_id() + previous = record.get('planned_at') if record.get('phase') == PHASE_PUBLISHING else None + planned_at = _message_timestamp(delivery.now()) + + def publishing(current, _now): + current.update({'phase': PHASE_PUBLISHING, 'message_id': message_id, 'planned_at': planned_at}) + return current + + _write_claimed(delivery, publishing) + try: + outcome, _conversation = services.mark_unread_guarded( + services.conversations, delivery.conversation_id, delivery.user_id, message_id, planned_at, + require_private=True, skip_if_read_since=previous if isinstance(previous, str) and previous else None, + ) + except Exception as exc: + delivery_log( + 'Unread mark failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('unread_unavailable') from None + if outcome in {UNREAD_NOT_FOUND, UNREAD_UNAVAILABLE}: + # The chat stopped taking deliveries between the re-check and the mark. + raise _Undeliverable(REASON_CHAT_UNAVAILABLE) + if outcome not in {UNREAD_MARKED, UNREAD_ALREADY_MARKED, UNREAD_READ_SINCE}: + raise _Transient('unread_conflict') + if outcome == UNREAD_MARKED: + try: + services.bump_cache_version(delivery.user_id, reason='workflow_chat_delivery') + except Exception as exc: + # A stale chat list only delays the unread dot; it never fails the delivery. + delivery_log( + 'Chat cache bump failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + def marked(current, _now): + current['phase'] = PHASE_UNREAD_MARKED + return current + + _write_claimed(delivery, marked) + + +def _create_message(delivery, build_message, *, reason=None): + """Place M after the chat's newest message and create it (D9 step 7). A 409 means it exists. + + ``reason`` is the closed code a note stands in for, kept on the record with the phase. + """ + services = delivery.services + message_id = delivery.message_id() + newest = _newest_message(delivery) + now = delivery.now() + newest_at = parse_delivery_timestamp(newest.get('timestamp')) if newest else None + moment = newest_at + timedelta(microseconds=1) if newest_at is not None and newest_at >= now else now + previous_thread = (newest.get('thread_info') or {}) if newest else {} + thread_info = { + 'thread_id': delivery_thread_id(message_id), + 'previous_thread_id': previous_thread.get('thread_id') if isinstance(previous_thread, dict) else None, + 'active_thread': True, + 'thread_attempt': 1, + } + document = build_message(_message_timestamp(moment), thread_info) + try: + services.messages.create_item(body=document) + except Exception as exc: + if _status_code(exc) != 409: + delivery_log( + 'Delivery message create failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('message_create_failed') from None + + def created(current, _now): + current.update({'phase': PHASE_MESSAGE_CREATED, 'message_id': message_id, 'outcome_reason': reason}) + return current + + _write_claimed(delivery, created) + + +class _TooLarge(Exception): + """Not even the smallest excerpt fits the model's context window.""" + + +class _ResultClosed(Exception): + """The reader closed the result for a reason a retry won't change.""" + + def __init__(self, code): + super().__init__(code) + self.code = code + + +_RESULT_CODE_PREFIX = 'workflow_result_' +_RESULT_RETRY_CODES = frozenset({ + 'workflow_result_storage_unavailable', + 'workflow_result_in_progress', + 'workflow_result_not_finished', +}) +_RESULT_MISSING_CODE = 'workflow_result_not_found' + + +@dataclass +class _Content: + """What this generation posts: the final text, already checked, and its metadata.""" + + kind: str + text: str + reason: Optional[str] = None + descriptor: Optional[dict] = None + contexts: Optional[list] = None + model_deployment_name: Optional[str] = None + token_usage: Optional[dict] = None + check: Any = None + blocked_check: Any = None + + +def _note(delivery, label, kind, *, reason=None, reason_code=None, descriptor=None, contexts=None): + body = delivery_note_text(kind, delivery.workflow_name(), reason_code=reason_code) + return _Content(kind, assemble_delivery_content(label, body), reason=reason, descriptor=descriptor, contexts=contexts) + + +def _read_result(delivery, budget, *, expected_sha256=None): + """Read the run's result at one excerpt budget, mapping the reader's closed reasons (D10).""" + services = delivery.services + try: + return services.read_workflow_result( + delivery.user_id, delivery.workflow_id, delivery.run_id, + expected_sha256=expected_sha256, include_excerpts=True, excerpt_budget_bytes=budget, + ) + except services.result_unavailable_error as exc: + code = getattr(exc, 'code', None) + code = code if isinstance(code, str) and code else 'workflow_result_invalid' + if code in _RESULT_RETRY_CODES: + # Storage was unavailable, or the run document hasn't caught up with the runtime yet. + raise _Transient(code[len(_RESULT_CODE_PREFIX):]) from None + raise _ResultClosed(code) from None + except Exception as exc: + delivery_log( + 'Workflow result read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('result_unavailable') from None + + +def _first_excerpt_text(result): + for item in result.get('excerpts') or []: + if isinstance(item, dict) and isinstance(item.get('text'), str) and item['text'].strip(): + return item['text'] + return '' + + +class _Excerpt: + """The result as last read, and the smaller excerpt budgets left to try (D10).""" + + def __init__(self, delivery, result): + self.delivery = delivery + self.result = result + steps = [step for step in delivery.services.excerpt_budget_steps if type(step) is int] + self.steps = [step for step in steps if 0 < step < EXCERPT_BUDGET_BYTES] + + def step_down(self): + if not self.steps: + raise _TooLarge() + descriptor = self.result.get('descriptor') if isinstance(self.result, dict) else None + expected = descriptor.get('result_sha256') if isinstance(descriptor, dict) else None + # Bound to the digest first read, so a smaller excerpt is always of the same result. + self.result = _read_result(self.delivery, self.steps.pop(0), expected_sha256=expected) + + +def _question(delivery): + """The request's own text, clipped; the fixed question when it's gone or was retracted.""" + services = delivery.services + message_id = delivery.invocation.get('user_message_id') + if not isinstance(message_id, str) or not message_id: + return FALLBACK_QUESTION + try: + message = services.messages.read_item(item=message_id, partition_key=delivery.conversation_id) + except Exception as exc: + if _status_code(exc) == 404: + return FALLBACK_QUESTION + delivery_log( + 'Request message read failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('message_unavailable') from None + if ( + not isinstance(message, dict) or message.get('role') != 'user' + or message.get('conversation_id') != delivery.conversation_id + or services.is_message_retracted(message) + ): + return FALLBACK_QUESTION + text = message.get('content') + text = text.strip() if isinstance(text, str) else '' + return text[:QUESTION_MAX_CHARS] if text else FALLBACK_QUESTION + + +def _model_metadata(model): + try: + metadata = model.metadata() + except Exception: + metadata = None + return metadata if isinstance(metadata, dict) else {} + + +def _close_model(delivery, model): + close = getattr(model, 'close', None) + if not callable(close): + return + try: + close() + except Exception as exc: + delivery_log( + 'Delivery model client close failed.', level=logging.INFO, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + +def _resolve_model(delivery, seeds): + """The model for ``seeds``, re-authorized for the requester now; None when it's unavailable.""" + services = delivery.services + try: + identity = services.capture_identity(delivery.user_id, conversation_id=delivery.conversation_id) + return services.resolve_model( + delivery.settings, user_id=delivery.user_id, seeds=seeds, identity_context=identity, + ) + except Exception as exc: + level = logging.INFO if isinstance(exc, services.model_error) else logging.WARNING + delivery_log( + 'Delivery model unavailable.', level=level, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return None + + +def _usage_from(response): + usage = getattr(response, 'usage', None) + values = {} + for field in ('prompt_tokens', 'completion_tokens', 'total_tokens'): + value = getattr(usage, field, None) if usage is not None else None + if type(value) is int and value >= 0: + values[field] = value + return values + + +def _reply_text(services, response): + """The reply, clipped; None for an empty, filtered or refused completion.""" + choices = getattr(response, 'choices', None) or [] + if not choices: + return None + choice = choices[0] + if getattr(choice, 'finish_reason', None) == 'content_filter': + return None + if getattr(getattr(choice, 'message', None), 'refusal', None): + return None + text = services.extract_text(response) + text = text.strip() if isinstance(text, str) else '' + return text[:REPLY_MAX_CHARS] if text else None + + +def _fits(services, model, messages): + """Whether the messages fit the model's window with room for the reply.""" + name = getattr(model, 'model_metadata', None) or getattr(model, 'behavior_name', None) or model.deployment + output_tokens = COMPOSE_MAX_TOKENS + if callable(services.compose_output_tokens): + output_tokens = services.compose_output_tokens(model) + budget = services.calculate_budget( + messages, name, provider=getattr(model, 'provider', None), output_tokens=output_tokens, + ) + return isinstance(budget, dict) and budget.get('decision') == 'full_input' + + +def _compose_with(delivery, model, excerpt, question, time_zone): + """One model's reply, stepping the excerpt down until it fits; None when it can't answer. + + Raises ``_TooLarge`` once the smallest excerpt still doesn't fit. + """ + services = delivery.services + while True: + try: + # A fresh nonce for every request, named by the system prompt that fences the excerpt. + messages = services.build_result_messages( + delivery.settings, excerpt.result, [], question, time_zone, nonce=services.new_fence_nonce(), + ) + fits = _fits(services, model, messages) + except Exception as exc: + delivery_log( + 'Delivery prompt could not be prepared for this model.', level=logging.WARNING, + run_id=delivery.run_id, error_type=type(exc).__name__, + ) + return None + if fits: + break + excerpt.step_down() + try: + response = model.create_completion( + messages=messages, temperature=COMPOSE_TEMPERATURE, max_tokens=COMPOSE_MAX_TOKENS, + ) + reply = _reply_text(services, response) + except Exception as exc: + delivery_log( + 'Delivery compose failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return None + if reply is None: + delivery_log('Delivery compose returned no usable reply.', level=logging.INFO, run_id=delivery.run_id) + return None + return reply, _usage_from(response) + + +def _compose(delivery, excerpt, question, time_zone): + """The reply from the request's model, then the default model (D10). + + Returns ``(reply, deployment, usage)``, or None when neither model answered. + """ + selection = delivery.record.get('model_selection') + chain = [] + if ( + isinstance(selection, dict) and isinstance(selection.get('model'), dict) + and any(selection['model'].values()) + ): + chain.append(selection) + chain.append({}) + tried = set() + for seeds in chain: + model = _resolve_model(delivery, seeds) + if model is None: + continue + try: + metadata = _model_metadata(model) + binding = ( + metadata.get('model_deployment_name'), metadata.get('model_provider'), + metadata.get('model_endpoint_id'), metadata.get('model_id'), + ) + if binding in tried: + continue + tried.add(binding) + composed = _compose_with(delivery, model, excerpt, question, time_zone) + if composed is not None: + reply, usage = composed + return reply, metadata.get('model_deployment_name') or getattr(model, 'deployment', None), usage + finally: + _close_model(delivery, model) + return None + + +def _log_compose_usage(delivery, deployment, usage): + """Count the compose the way 6a counts a Follow up answer (A4). Logging never fails delivery.""" + total = usage.get('total_tokens') if isinstance(usage, dict) else None + if not total: + return + message_id = delivery.message_id() + try: + delivery.services.log_token_usage( + user_id=delivery.user_id, token_type='chat', conversation_id=delivery.conversation_id, + message_id=message_id, model=deployment, workspace_type='personal', total_tokens=total, + prompt_tokens=usage.get('prompt_tokens'), completion_tokens=usage.get('completion_tokens'), + additional_context={'workflow_result_delivery': True}, + idempotency_key=token_usage_idempotency_key(message_id, delivery.record.get('attempts')), + ) + except Exception as exc: + delivery_log( + 'Delivery token usage logging failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + +def _fenced(text): + """A code block whose fence is longer than any run of backticks in the text.""" + longest = max((len(run) for run in re.findall(r'`+', text)), default=0) + fence = '`' * max(3, longest + 1) + return f'{fence}\n{text}\n{fence}' + + +def _disclosure(delivery, result, time_zone, *, truncated=False): + return delivery.services.format_disclosure( + result.get('descriptor') or {}, time_zone, + truncated=truncated or bool(result.get('truncated')), partial=bool(result.get('partial')), + analysis_only=bool(result.get('analysis_only')), skipped_reports=bool(result.get('skipped_reports')), + ) + + +def _checked(delivery, label, content): + """Check the final text as chat output; a blocked reply posts the fixed note instead (D10).""" + services = delivery.services + try: + check = services.check_content( + content.text, 'chat_output', user_id=delivery.user_id, settings=delivery.settings, + ) + except Exception as exc: + delivery_log( + 'Delivery content check failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('content_check_unavailable') from None + if getattr(check, 'blocked', False): + blocked = _note(delivery, label, KIND_CONTENT_BLOCKED, reason=REASON_CONTENT_BLOCKED) + blocked.blocked_check = check + return blocked + content.check = check + return content + + +def _closed_result(delivery, label, code): + """A result the reader closed: the status note, or no post at all when the run is gone.""" + if code == _RESULT_MISSING_CODE: + if _read_workflow(delivery) is None: + raise _Undeliverable(REASON_WORKFLOW_DELETED) + raise _Undeliverable(REASON_ACCESS_LOST) + delivery_log('Workflow result closed; posting the status note.', level=logging.INFO, run_id=delivery.run_id, code=code) + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + + +def _result_content(delivery, label, time_zone): + """The composed reply for a completed run, or the note that stands in for it (D10).""" + services = delivery.services + if not services.is_chat_workflow_results_enabled(delivery.settings, user_roles=delivery.record.get('requester_roles')): + # Without Use Workflow Results In Chat, nothing of the result reaches a model or the chat. + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULTS_OFF) + try: + result = _read_result(delivery, EXCERPT_BUDGET_BYTES) + descriptor = dict(result.get('descriptor') or {}) + contexts = [services.workflow_result_context(descriptor)] + if result.get('saved_inputs'): + return _note(delivery, label, KIND_ANALYSIS, descriptor=descriptor, contexts=contexts) + if not _first_excerpt_text(result): + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + excerpt = _Excerpt(delivery, result) + composed = _compose(delivery, excerpt, _question(delivery), time_zone) + except _ResultClosed as exc: + return _closed_result(delivery, label, exc.code) + except _TooLarge: + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + except services.result_unavailable_error: + # The descriptor didn't name a valid result context, so Follow up couldn't use it. + return _note(delivery, label, KIND_STATUS, reason=REASON_RESULT_UNAVAILABLE) + result = excerpt.result + if composed is None: + # Neither model answered: the start of the result, fenced, under a fixed sentence. + body = f'{TRIMMED_FALLBACK_INTRO}\n\n{_fenced(_first_excerpt_text(result)[:TRIMMED_FALLBACK_CHARS])}' + text = assemble_delivery_content(label, body, _disclosure(delivery, result, time_zone, truncated=True)) + content = _Content(KIND_RESULT, text, descriptor=descriptor, contexts=contexts) + else: + reply, deployment, usage = composed + _log_compose_usage(delivery, deployment, usage) + text = assemble_delivery_content(label, reply, _disclosure(delivery, result, time_zone)) + content = _Content( + KIND_RESULT, text, descriptor=descriptor, contexts=contexts, + model_deployment_name=deployment, token_usage=usage, + ) + return _checked(delivery, label, content) + + +def _failure_code(delivery): + """The closed reason code for a failed run: the runtime's gate reason, else its state.""" + try: + summary = _read_control_summary(delivery.services, delivery.user_id, delivery.run) + except _Transient: + summary = {} + return summary.get('gate_reason_code') or delivery.record.get('run_status') + + +def _content(delivery, label, time_zone): + kind = delivery.record.get('kind') + if kind == KIND_RESULT: + return _result_content(delivery, label, time_zone) + if kind == KIND_FAILED: + return _note(delivery, label, KIND_FAILED, reason_code=_failure_code(delivery)) + if kind in {KIND_CANCELLED, KIND_SKIPPED}: + return _note(delivery, label, kind) + raise _Transient('unsupported_kind') + + +def _message_builder(delivery, content): + """Build the delivery message for the timestamp and thread placement chosen at create time.""" + services = delivery.services + record = delivery.record + invocation = delivery.invocation + message_id = delivery.message_id() + + def build(timestamp, thread_info): + metadata = { + DELIVERY_METADATA_KEY: build_delivery_metadata( + kind=content.kind, workflow_id=delivery.workflow_id, run_id=delivery.run_id, + generation=record.get('generation'), run_status=record.get('run_status'), + orchestration_run_id=invocation.get('orchestration_run_id'), step_id=invocation.get('step_id'), + requested_at=invocation.get('requested_at'), + ), + 'token_usage': dict(content.token_usage or {}), + 'user_info': {'user_id': delivery.user_id}, + 'thread_info': thread_info, + } + if content.kind in RESULT_CONTEXT_KINDS and content.descriptor and content.contexts: + # The same lineage keys a 6a answer carries, so Follow up and masking treat it alike. + metadata['workflow_result'] = dict(content.descriptor) + metadata['workflow_result_contexts'] = [dict(item) for item in content.contexts] + document = { + 'id': message_id, + 'conversation_id': delivery.conversation_id, + 'role': 'assistant', + 'content': content.text, + 'timestamp': timestamp, + 'model_deployment_name': content.model_deployment_name, + 'augmented': False, + 'hybrid_citations': [], + 'hybridsearch_query': None, + 'agent_citations': [], + 'web_search_citations': [], + 'user_message': None, + 'metadata': metadata, + } + if content.check is not None: + services.attach_chat_check(document, content.check) + return document + + return build + + +def _record_incident(delivery, check): + """Best-effort incident for a blocked reply. The record holds categories, never text.""" + services = delivery.services + try: + stub = services.attach_chat_check( + {'id': delivery.message_id(), 'conversation_id': delivery.conversation_id}, check, + ) + services.record_incident(stub, delivery.user_id) + except Exception as exc: + delivery_log( + 'Delivery content incident failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + + +SWEEP_QUERY = ( + 'SELECT TOP @top c.id, c.user_id FROM c ' + 'WHERE IS_DEFINED(c.chat_delivery) ' + 'AND (NOT IS_DEFINED(c.chat_delivery.next_attempt_at) OR IS_NULL(c.chat_delivery.next_attempt_at) ' + 'OR c.chat_delivery.next_attempt_at <= @now) ' + 'AND (c.chat_delivery.status = @ready ' + 'OR (c.chat_delivery.status = @delivering AND c.chat_delivery.lease_expires_at <= @now) ' + 'OR (c.chat_delivery.status = @pending AND (c.chat_delivery.expires_at <= @now ' + 'OR (ARRAY_CONTAINS(@terminal, c.status) AND c._ts <= @grace_ts))))' +) + +_OUTCOME_BY_STATUS = { + STATUS_DELIVERED: OUTCOME_DELIVERED, + STATUS_UNDELIVERABLE: OUTCOME_UNDELIVERABLE, + STATUS_EXPIRED: OUTCOME_EXPIRED, +} + + +def _write_phase(delivery, phase): + def advance(current, _now): + current['phase'] = phase + return current + + _write_claimed(delivery, advance) + + +def _signal(delivery): + try: + delivery.services.signal_delivery(delivery.user_id, delivery.run_id) + except Exception as exc: + delivery_log('Delivery hint failed.', level=logging.INFO, run_id=delivery.run_id, error_type=type(exc).__name__) + + +def _finish(delivery, status, notice_kind, *, reason=None): + """Close the generation and release the lease. + + The write is reconciled with a fresh control read, so a resume that landed while this + generation was being delivered reopens the record for the next one. + """ + try: + summary = _read_control_summary(delivery.services, delivery.user_id, delivery.run) + except _Transient: + # The projection of the resume's own terminal write reconciles the record instead. + summary = None + + def close(current, now): + current.update({ + 'status': status, + 'notice_kind': notice_kind, + 'next_attempt_at': None, + 'first_deferred_at': None, + }) + if status == STATUS_DELIVERED: + current['delivered_at'] = current.get('delivered_at') or format_delivery_timestamp(now) + if reason is not None: + current['outcome_reason'] = reason + return current + + record = _write_claimed(delivery, close, release=True, summary=summary) + if record.get('status') == STATUS_READY: + _signal(delivery) + delivery_log( + 'Delivery closed.', run_id=delivery.run_id, conversation_id=delivery.conversation_id, + status=status, reason=reason, + ) + return _OUTCOME_BY_STATUS.get(status, OUTCOME_NOT_APPLICABLE) + + +def _close_silently(delivery, reason): + """Close without a post or a notice: the run itself is gone (A1).""" + + def close(current, now): + return _close_for_runtime(current, reason, format_delivery_timestamp(now)) + + _write_claimed(delivery, close, release=True) + delivery_log('Delivery closed silently.', run_id=delivery.run_id, reason=reason) + return OUTCOME_CLOSED_SILENTLY + + +def _conversation_title(delivery): + conversation = delivery.conversation + if conversation is None and isinstance(delivery.conversation_id, str) and delivery.conversation_id: + try: + conversation = delivery.services.conversations.read_item( + item=delivery.conversation_id, partition_key=delivery.conversation_id, + ) + except Exception as exc: + delivery_log( + 'Chat title read failed.', level=logging.INFO, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + conversation = None + if not isinstance(conversation, dict) or conversation.get('user_id') != delivery.user_id: + return '' + title = conversation.get('title') + return title if isinstance(title, str) else '' + + +def _send_chat_notice(delivery): + """The bell notice for a posted message. It opens the chat; its key makes it one per generation.""" + try: + delivery.services.create_chat_response_notification( + delivery.user_id, delivery.conversation_id, delivery.message_id(), + conversation_title=_conversation_title(delivery), + response_preview=delivered_notice_preview(delivery.workflow_name()), + idempotency_key=delivery_notification_key(delivery.run_id, delivery.generation), + strict=True, + ) + except Exception as exc: + delivery_log( + 'Delivery notice failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('notification_failed') from None + + +def _send_notice(delivery, kind, key, message, delivery_status): + """One fixed workflow notice that links to the run. Nothing in it comes from the result.""" + name = delivery.workflow_name() + try: + title = notice_title(kind, name) + except ValueError: + title = notice_title(KIND_STATUS, name) + try: + delivery.services.create_notification( + user_id=delivery.user_id, + notification_type=NOTIFICATION_TYPE, + title=title, + message=message, + link_url=workflow_run_notice_link(delivery.workflow_id, delivery.run_id), + metadata=notice_metadata(delivery.workflow_id, delivery.run_id, delivery_status), + idempotency_key=key, + strict=True, + ) + except Exception as exc: + delivery_log( + 'Delivery notice failed.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + raise _Transient('notification_failed') from None + + +def _notify_delivered(delivery): + if not phase_at_least(delivery.record.get('phase'), PHASE_NOTIFIED): + _send_chat_notice(delivery) + _write_phase(delivery, PHASE_NOTIFIED) + return _finish(delivery, STATUS_DELIVERED, NOTICE_CHAT_RESPONSE) + + +def _undeliverable(delivery, reason): + """Post nothing; send one notice that says the chat can't show the run, unless it's silent. + + The notice's key makes it one per generation, so this path never records ``notified``: that + phase means a message exists. + """ + if reason in SILENT_REASONS: + return _close_silently(delivery, reason) + if phase_at_least(delivery.record.get('phase'), PHASE_MESSAGE_CREATED): + return _notify_delivered(delivery) + _send_notice( + delivery, delivery.record.get('kind'), + undeliverable_notification_key(delivery.run_id, delivery.generation), + UNDELIVERABLE_NOTICE_MESSAGE, STATUS_UNDELIVERABLE, + ) + return _finish(delivery, STATUS_UNDELIVERABLE, NOTICE_UNDELIVERABLE, reason=reason) + + +def _expire(delivery): + """The run didn't finish before the delivery window closed: one notice, nothing in the chat.""" + _send_notice( + delivery, KIND_EXPIRED, expired_notification_key(delivery.run_id), + EXPIRED_NOTICE_MESSAGE, STATUS_EXPIRED, + ) + return _finish(delivery, STATUS_EXPIRED, NOTICE_EXPIRED) + + +def _exhausted(delivery, reason): + """Attempts ran out. Close with a best-effort notice; never post a second message.""" + phase = delivery.record.get('phase') + created = phase_at_least(phase, PHASE_MESSAGE_CREATED) + if not created and phase_at_least(phase, PHASE_PUBLISHING): + try: + created = _message_exists(delivery, delivery.message_id()) + except _Transient: + # Unknown: an undeliverable notice beats a silent loss. + created = False + if created: + try: + _send_chat_notice(delivery) + notice = NOTICE_CHAT_RESPONSE + except _Transient: + notice = NOTICE_NONE + return _finish(delivery, STATUS_DELIVERED, notice) + if delivery.record.get('kind') == KIND_EXPIRED: + # An expiry whose notice kept failing still closes as an expiry. + try: + _send_notice( + delivery, KIND_EXPIRED, expired_notification_key(delivery.run_id), + EXPIRED_NOTICE_MESSAGE, STATUS_EXPIRED, + ) + notice = NOTICE_EXPIRED + except _Transient: + notice = NOTICE_NONE + return _finish(delivery, STATUS_EXPIRED, notice) + try: + _send_notice( + delivery, delivery.record.get('kind'), + undeliverable_notification_key(delivery.run_id, delivery.generation), + UNDELIVERABLE_NOTICE_MESSAGE, STATUS_UNDELIVERABLE, + ) + notice = NOTICE_UNDELIVERABLE + except _Transient: + notice = NOTICE_NONE + return _finish(delivery, STATUS_UNDELIVERABLE, notice, reason=reason) + + +def _retry_later(delivery, reason): + attempts = _count(delivery.record.get('attempts')) + if attempts >= MAX_ATTEMPTS: + return _exhausted(delivery, REASON_DELIVERY_FAILED) + delay = next_backoff_seconds(attempts) + + def later(current, now): + current.update({'status': STATUS_READY, 'next_attempt_at': _later(now, delay)}) + return current + + _write_claimed(delivery, later, release=True) + delivery_log( + 'Delivery attempt failed; it will be retried.', level=logging.WARNING, run_id=delivery.run_id, + reason=reason, attempt=attempts, + ) + return OUTCOME_RETRY_SCHEDULED + + +def _defer(delivery, counted): + """Wait for the chat to settle (D7). An uncounted deferral gives the claim's attempt back. + + A counted deferral (the chat's activity couldn't be read) backs off like any failed attempt. + """ + attempts = _count(delivery.record.get('attempts')) + if counted and attempts >= MAX_ATTEMPTS: + return _exhausted(delivery, REASON_DELIVERY_FAILED) + delay = next_backoff_seconds(attempts) if counted else DEFER_RETRY_SECONDS + + def deferred(current, now): + current.update({'status': STATUS_READY, 'next_attempt_at': _later(now, delay)}) + if not counted: + current['attempts'] = max(0, _count(current.get('attempts')) - 1) + if not current.get('first_deferred_at'): + current['first_deferred_at'] = format_delivery_timestamp(now) + return current + + _write_claimed(delivery, deferred, release=True) + delivery_log('Delivery deferred while the chat is busy.', run_id=delivery.run_id, step='counted' if counted else 'activity') + return OUTCOME_DEFERRED + + +def _asked_when(delivery, time_zone): + try: + asked = delivery.services.format_run_time(delivery.invocation.get('requested_at'), time_zone) + except Exception as exc: + delivery_log('Request time could not be formatted.', level=logging.INFO, run_id=delivery.run_id, error_type=type(exc).__name__) + return '' + return asked if isinstance(asked, str) else '' + + +def _deliver(delivery): + """One claimed attempt at the generation, resuming from the phase the last attempt reached (D9).""" + services = delivery.services + if delivery.exhausted: + return _exhausted(delivery, REASON_DELIVERY_FAILED) + try: + delivery.settings = services.get_settings() or {} + except Exception as exc: + delivery_log('Settings read failed.', level=logging.WARNING, run_id=delivery.run_id, error_type=type(exc).__name__) + raise _Transient('settings_unavailable') from None + delivery.workflow = _read_workflow(delivery) + if delivery.workflow is None: + return _close_silently(delivery, REASON_WORKFLOW_DELETED) + + phase = delivery.record.get('phase') + if ( + phase_at_least(phase, PHASE_PUBLISHING) and not phase_at_least(phase, PHASE_MESSAGE_CREATED) + and _message_exists(delivery, delivery.message_id()) + ): + # The create landed but its phase write didn't. + _write_phase(delivery, PHASE_MESSAGE_CREATED) + if phase_at_least(delivery.record.get('phase'), PHASE_MESSAGE_CREATED): + return _notify_delivered(delivery) + record = delivery.record + if record.get('kind') == KIND_EXPIRED: + return _expire(delivery) + + if _read_conversation(delivery) is None: + raise _Undeliverable(REASON_CHAT_UNAVAILABLE) + if not services.is_user_workflows_enabled(delivery.settings, user_roles=record.get('requester_roles')): + raise _Undeliverable(REASON_ACCESS_LOST) + now = delivery.now() + if not phase_at_least(record.get('phase'), PHASE_PUBLISHING): + expires_at = parse_delivery_timestamp(record.get('expires_at')) + if expires_at is not None and now > expires_at: + raise _Undeliverable(REASON_EXPIRED_BEFORE_DELIVERY) + _defer_probe(delivery, record, now) + + time_zone = record.get('time_zone') + label = delivery_label(delivery.workflow_name(), _asked_when(delivery, time_zone)) + content = _content(delivery, label, time_zone) + if not phase_at_least(delivery.record.get('phase'), PHASE_UNREAD_MARKED): + _mark_unread(delivery) + _create_message(delivery, _message_builder(delivery, content), reason=content.reason) + if content.blocked_check is not None: + _record_incident(delivery, content.blocked_check) + return _notify_delivered(delivery) + + +def _handle(delivery, handler, *args): + """Record the attempt's outcome. If even that fails, the lease lapses and a later claim retries.""" + try: + try: + return handler(delivery, *args) + except _Transient as exc: + if handler is not _undeliverable: + raise + return _retry_later(delivery, exc.reason) + except _LostLease: + return OUTCOME_LOST_LEASE + except Exception as exc: + delivery_log( + 'Delivery outcome could not be recorded.', level=logging.WARNING, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return OUTCOME_UNAVAILABLE + + +def _run_claimed(delivery): + try: + return _deliver(delivery) + except _LostLease: + return OUTCOME_LOST_LEASE + except _Undeliverable as exc: + return _handle(delivery, _undeliverable, exc.reason) + except _Defer as exc: + return _handle(delivery, _defer, exc.counted) + except _Transient as exc: + return _handle(delivery, _retry_later, exc.reason) + except Exception as exc: + delivery_log( + 'Delivery attempt failed unexpectedly.', level=logging.ERROR, run_id=delivery.run_id, + error_type=type(exc).__name__, + ) + return _handle(delivery, _retry_later, 'unexpected_error') + + +def _services_or_default(services): + if services is not None: + return services + try: + return default_workflow_chat_delivery_services() + except Exception as exc: + delivery_log('Delivery services unavailable.', level=logging.ERROR, error_type=type(exc).__name__) + return None + + +def process_workflow_chat_delivery(user_id, run_id, *, services=None): + """Deliver one run's outcome when it's due. Returns an ``OUTCOME_*`` string and never raises.""" + if not isinstance(user_id, str) or not user_id or not isinstance(run_id, str) or not run_id: + return OUTCOME_NOT_APPLICABLE + services = _services_or_default(services) + if services is None: + return OUTCOME_UNAVAILABLE + try: + claimed = _claim(services, user_id, run_id) + except _LostLease: + return OUTCOME_LOST_LEASE + except Exception as exc: + delivery_log('Delivery claim failed.', level=logging.WARNING, run_id=run_id, error_type=type(exc).__name__) + return OUTCOME_UNAVAILABLE + if not isinstance(claimed, _Delivery): + return claimed + return _run_claimed(claimed) + + +def process_workflow_chat_delivery_hints(services=None, limit=HINT_BATCH): + """Process this process's queued hints. They need no lock: every claim is an ETag write.""" + services = _services_or_default(services) + if services is None: + return [] + try: + hints = list(services.drain_hints(limit)) + except Exception as exc: + delivery_log('Delivery hints could not be read.', level=logging.WARNING, error_type=type(exc).__name__) + return [] + outcomes = [] + for user_id, run_id in hints: + outcomes.append((run_id, process_workflow_chat_delivery(user_id, run_id, services=services))) + return outcomes + + +def run_workflow_chat_delivery_sweep(services=None): + """Find due deliveries across all users and process a bounded batch under the sweep lock (D8). + + Returns ``{'locked', 'processed', 'outcomes'}``. Without the lock nothing is queried. + """ + summary = {'locked': False, 'processed': 0, 'outcomes': {}} + services = _services_or_default(services) + if services is None: + return summary + try: + lock = services.acquire_lock(SWEEP_LOCK_NAME, SWEEP_LOCK_SECONDS) + except Exception as exc: + delivery_log('Delivery sweep lock failed.', level=logging.WARNING, error_type=type(exc).__name__) + return summary + if not lock: + return summary + summary['locked'] = True + started = services.monotonic() + try: + now = _now(services) + rows = services.runs.query_items( + query=SWEEP_QUERY, + parameters=[ + {'name': '@top', 'value': SWEEP_TOP}, + {'name': '@now', 'value': format_delivery_timestamp(now)}, + {'name': '@ready', 'value': STATUS_READY}, + {'name': '@delivering', 'value': STATUS_DELIVERING}, + {'name': '@pending', 'value': STATUS_PENDING}, + {'name': '@terminal', 'value': sorted(RUN_DOCUMENT_TERMINAL_STATUSES)}, + {'name': '@grace_ts', 'value': int(now.timestamp()) - SWEEP_TS_GRACE_SECONDS}, + ], + enable_cross_partition_query=True, + ) + for row in rows: + if summary['processed'] >= SWEEP_MAX_RUNS or services.monotonic() - started >= SWEEP_MAX_SECONDS: + break + if not isinstance(row, dict): + continue + outcome = process_workflow_chat_delivery(row.get('user_id'), row.get('id'), services=services) + summary['processed'] += 1 + summary['outcomes'][outcome] = summary['outcomes'].get(outcome, 0) + 1 + except Exception as exc: + delivery_log('Delivery sweep failed.', level=logging.WARNING, error_type=type(exc).__name__) + finally: + try: + services.release_lock(lock) + except Exception as exc: + delivery_log('Delivery sweep lock release failed.', level=logging.WARNING, error_type=type(exc).__name__) + return summary + + +__all__ = ( + 'HINT_BATCH', + 'OUTCOME_BUSY', + 'OUTCOME_CLOSED_SILENTLY', + 'OUTCOME_DEFERRED', + 'OUTCOME_DELIVERED', + 'OUTCOME_EXPIRED', + 'OUTCOME_LOST_LEASE', + 'OUTCOME_NOT_APPLICABLE', + 'OUTCOME_NOT_DUE', + 'OUTCOME_PENDING', + 'OUTCOME_RETRY_SCHEDULED', + 'OUTCOME_UNAVAILABLE', + 'OUTCOME_UNDELIVERABLE', + 'QUESTION_MAX_CHARS', + 'SWEEP_QUERY', + 'WorkflowChatDeliveryServices', + 'default_workflow_chat_delivery_services', + 'process_workflow_chat_delivery', + 'process_workflow_chat_delivery_hints', + 'run_workflow_chat_delivery_sweep', +) diff --git a/functional_tests/test_support/workflow_chat_delivery_fakes.py b/functional_tests/test_support/workflow_chat_delivery_fakes.py new file mode 100644 index 000000000..9848b4419 --- /dev/null +++ b/functional_tests/test_support/workflow_chat_delivery_fakes.py @@ -0,0 +1,955 @@ +# workflow_chat_delivery_fakes.py +""" +Reusable fakes for the workflow chat delivery tests. + +The fakes stand in for Cosmos containers, the durable runtime store, the result +reader, the chat model and the notification helpers. The delivery contract and the +worker under test stay real. Checks inside this module raise explicitly so they +still run under ``python -O``. +""" + +import copy +import itertools +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from azure.cosmos.exceptions import ( + CosmosAccessConditionFailedError, + CosmosHttpResponseError, + CosmosResourceExistsError, + CosmosResourceNotFoundError, +) + +from functions_workflow_chat_delivery import ( + build_chat_delivery_seed, + finalize_chat_delivery_seed, +) +from functions_conversation_unread import mark_conversation_unread_guarded +from functions_orchestration_memory import conversation_is_private +from functions_workflow_chat_delivery_worker import WorkflowChatDeliveryServices + + +USER = "user-delivery-1" +OTHER_USER = "user-delivery-2" +WORKFLOW_ID = "workflow-delivery-1" +RUN_ID = "run-delivery-1" +CONVERSATION_ID = "conversation-delivery-1" +USER_MESSAGE_ID = "message-user-1" +ORCHESTRATION_RUN_ID = "orchestration-run-1" +ATTEMPT_ROOT_RUN_ID = "orchestration-run-1" +STEP_ID = "step-1" +REQUESTED_AT = "2026-05-04T14:05:00+00:00" +NOW = "2026-05-04T15:00:00+00:00" + + +class FakeCheckFailed(AssertionError): + """A fake saw a call it was not prepared for.""" + + +def require(condition, message): + if not condition: + raise FakeCheckFailed(message) + + +_COSMOS_ERRORS = { + 404: CosmosResourceNotFoundError, + 409: CosmosResourceExistsError, + 412: CosmosAccessConditionFailedError, +} + + +def cosmos_error(status_code, message="fake cosmos error"): + error_type = _COSMOS_ERRORS.get(status_code, CosmosHttpResponseError) + error = error_type(status_code=status_code, message=message) + error.status_code = status_code + return error + + +def iso(value): + return value.astimezone(timezone.utc).isoformat() + + +def parse_iso(value): + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + + +def shift(value, **delta): + return iso(parse_iso(value) + timedelta(**delta)) + + +class FakeClock: + """A settable ISO clock; ``advance`` moves it forward.""" + + def __init__(self, start=NOW): + self.now = start + + def __call__(self): + return self.now + + def advance(self, **delta): + self.now = shift(self.now, **delta) + return self.now + + +class FakeContainer: + """An in-memory container with ETag, 404, 409 and 412 semantics.""" + + def __init__(self, name, items=None, query_handler=None, clock=None): + self.name = name + self.items = {} + self.calls = [] + self.query_calls = [] + self.query_handler = query_handler + self.clock = clock + self.fail_next = {} + self._etags = itertools.count(1) + for item in items or []: + self.put(item) + + def _key(self, item_id): + return str(item_id) + + def _stamp(self, body): + stored = copy.deepcopy(body) + stored["_etag"] = f"etag-{self.name}-{next(self._etags)}" + if self.clock is not None: + stored["_ts"] = int(parse_iso(self.clock()).timestamp()) + else: + stored.setdefault("_ts", 1777900000) + return stored + + def put(self, body): + stored = self._stamp(body) + self.items[self._key(stored["id"])] = stored + return copy.deepcopy(stored) + + def get(self, item_id): + stored = self.items.get(self._key(item_id)) + return copy.deepcopy(stored) if stored is not None else None + + def fail(self, operation, *errors): + """Queue errors for the next calls of ``operation``.""" + self.fail_next.setdefault(operation, []).extend(errors) + + def _maybe_fail(self, operation): + queued = self.fail_next.get(operation) or [] + if queued: + error = queued.pop(0) + if callable(error) and not isinstance(error, BaseException): + error = error() + if error is not None: + raise error + + def read_item(self, item=None, partition_key=None, **kwargs): + self.calls.append(("read_item", item, partition_key)) + self._maybe_fail("read_item") + stored = self.items.get(self._key(item)) + if stored is None: + raise cosmos_error(404, "not found") + return copy.deepcopy(stored) + + def replace_item(self, item=None, body=None, etag=None, match_condition=None, **kwargs): + item_id = item.get("id") if isinstance(item, dict) else item + self.calls.append(("replace_item", item_id, etag)) + self._maybe_fail("replace_item") + stored = self.items.get(self._key(item_id)) + if stored is None: + raise cosmos_error(404, "not found") + if etag is not None and stored.get("_etag") != etag: + raise cosmos_error(412, "precondition failed") + updated = self._stamp(body) + self.items[self._key(item_id)] = updated + return copy.deepcopy(updated) + + def create_item(self, body=None, **kwargs): + self.calls.append(("create_item", body.get("id"))) + self._maybe_fail("create_item") + if self._key(body.get("id")) in self.items: + raise cosmos_error(409, "conflict") + return self.put(body) + + def upsert_item(self, body=None, **kwargs): + self.calls.append(("upsert_item", body.get("id"))) + self._maybe_fail("upsert_item") + return self.put(body) + + def delete_item(self, item=None, partition_key=None, **kwargs): + item_id = item.get("id") if isinstance(item, dict) else item + self.calls.append(("delete_item", item_id)) + self._maybe_fail("delete_item") + if self.items.pop(self._key(item_id), None) is None: + raise cosmos_error(404, "not found") + + def query_items(self, query=None, parameters=None, partition_key=None, + enable_cross_partition_query=None, **kwargs): + self.query_calls.append({ + "query": query, + "parameters": copy.deepcopy(parameters or []), + "partition_key": partition_key, + "enable_cross_partition_query": enable_cross_partition_query, + }) + self._maybe_fail("query_items") + if self.query_handler is None: + raise FakeCheckFailed(f"{self.name}: unexpected query {query!r}") + params = {entry["name"]: entry["value"] for entry in parameters or []} + rows = self.query_handler(self, query, params, partition_key) + return iter([copy.deepcopy(row) for row in rows]) + + def writes(self, operation=None): + names = ("replace_item", "create_item", "upsert_item", "delete_item") + return [call for call in self.calls if call[0] in names and (operation is None or call[0] == operation)] + + +class FakeRuntimeConflict(Exception): + def __init__(self, code="conflict"): + super().__init__(code) + self.code = code + + +class FakeRuntimeUnavailable(Exception): + pass + + +class FakeResultUnavailable(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +class FakeModelError(Exception): + pass + + +class FakeRuntime: + """Runtime controls by run ID. ``factory`` returns a store whose ``read`` honors tombstones.""" + + def __init__(self): + self.controls = {} + self.errors = {} + self.reads = [] + + def set(self, run_id=RUN_ID, *, state="completed", version=5, deleted=False, gate_reason_code=None, + deadline_at=None, phase=None, schema_version=2): + control = {"state": state, "version": version, "deleted": deleted, "schema_version": schema_version} + if phase is not None: + control["phase"] = phase + if gate_reason_code is not None: + control["gate"] = {"reason_code": gate_reason_code} + if deadline_at is not None: + control["limits"] = {"deadline_at": deadline_at} + self.controls[run_id] = control + return copy.deepcopy(control) + + def update(self, run_id=RUN_ID, **changes): + control = self.controls[run_id] + control.update(changes) + return copy.deepcopy(control) + + def tombstone(self, run_id=RUN_ID): + control = self.controls[run_id] + control.update({"deleted": True, "state": "cancelled", "version": control["version"] + 1}) + return copy.deepcopy(control) + + def remove(self, run_id=RUN_ID): + self.controls.pop(run_id, None) + + def fail(self, run_id, *errors): + self.errors.setdefault(run_id, []).extend(errors) + + def factory(self, user_id, workflow_id, run_id): + runtime = self + + class _Store: + def read(self, allow_deleted=False): + runtime.reads.append((user_id, workflow_id, run_id, allow_deleted)) + queued = runtime.errors.get(run_id) or [] + if queued: + raise queued.pop(0) + control = runtime.controls.get(run_id) + if control is None or (control.get("deleted") and not allow_deleted): + raise FakeRuntimeConflict("not_found") + return copy.deepcopy(control) + + return _Store() + + +RESULT_SHA = "a" * 64 +RESULT_TEXT = "The digest found three new files and one changed report." + + +def make_result(*, text=RESULT_TEXT, budget=None, saved_inputs=None, truncated=False, partial=False, + analysis_only=False, skipped_reports=False, sha=RESULT_SHA): + descriptor = { + "workflow_id": WORKFLOW_ID, + "run_id": RUN_ID, + "result_sha256": sha, + "workflow_name": "Daily digest", + "status": "completed", + "completed_at": "2026-05-04T14:30:00+00:00", + } + return { + "descriptor": descriptor, + "excerpts": [{"text": text}] if text else [], + "truncated": truncated, + "partial": partial, + "analysis_only": analysis_only, + "skipped_reports": skipped_reports, + "saved_inputs": list(saved_inputs or []), + "budget": budget, + } + + +class FakeResultReader: + """``read_workflow_result``: queued outcomes first, then the default result at the asked budget.""" + + def __init__(self): + self.calls = [] + self.outcomes = [] + self.text = RESULT_TEXT + self.saved_inputs = None + + def __call__(self, user_id, workflow_id, run_id, *, expected_sha256=None, include_excerpts=True, + excerpt_budget_bytes=None): + self.calls.append({ + "user_id": user_id, + "workflow_id": workflow_id, + "run_id": run_id, + "expected_sha256": expected_sha256, + "include_excerpts": include_excerpts, + "budget": excerpt_budget_bytes, + }) + if self.outcomes: + outcome = self.outcomes.pop(0) + if isinstance(outcome, BaseException): + raise outcome + if callable(outcome): + return outcome(excerpt_budget_bytes) + return copy.deepcopy(outcome) + return make_result(text=self.text, budget=excerpt_budget_bytes, saved_inputs=self.saved_inputs) + + +def fake_workflow_result_context(descriptor): + if not isinstance(descriptor, dict) or not descriptor.get("result_sha256"): + raise FakeResultUnavailable("workflow_result_invalid") + return { + "workflow_id": descriptor["workflow_id"], + "run_id": descriptor["run_id"], + "result_sha256": descriptor["result_sha256"], + } + + +def fake_format_run_time(value, time_zone): + if not value: + return "" + return f"May 4, 2026 at 10:05 AM ({time_zone})" + + +class FakeDisclosure: + def __init__(self): + self.calls = [] + + def __call__(self, descriptor, time_zone, *, truncated=False, partial=False, analysis_only=False, + skipped_reports=False): + self.calls.append({ + "descriptor": copy.deepcopy(descriptor), + "time_zone": time_zone, + "truncated": truncated, + "partial": partial, + "analysis_only": analysis_only, + "skipped_reports": skipped_reports, + }) + flags = [name for name, value in (("truncated", truncated), ("partial", partial)) if value] + suffix = f" ({', '.join(flags)})" if flags else "" + return f"Source: workflow run from May 4, 2026{suffix}." + + +class FakePromptBuilder: + """``build_workflow_result_messages``: records the excerpt budget, question and nonce.""" + + def __init__(self): + self.calls = [] + self.error = None + + def __call__(self, settings, result, history, question, time_zone, *, nonce): + self.calls.append({"budget": result.get("budget"), "question": question, "nonce": nonce, "time_zone": time_zone}) + if self.error is not None: + raise self.error + excerpt = (result.get("excerpts") or [{}])[0].get("text", "") + return [ + {"role": "system", "content": f"Answer from the fenced result. Fence code: {nonce}.", "budget": result.get("budget")}, + {"role": "user", "content": f"<<{nonce}>>\n{excerpt}\n<>\n\n{question}"}, + ] + + +class FakeNonces: + def __init__(self): + self.issued = [] + + def __call__(self): + nonce = f"nonce-{len(self.issued) + 1}" + self.issued.append(nonce) + return nonce + + +class FakeBudget: + """``calculate_workflow_context_budget``: fits when the excerpt budget is at most ``max_budget``.""" + + def __init__(self): + self.calls = [] + self.max_budget = None + + def __call__(self, messages, model_name, provider=None, output_tokens=None): + budget = messages[0].get("budget") if messages else None + self.calls.append({"budget": budget, "model": model_name, "output_tokens": output_tokens}) + fits = self.max_budget is None or (budget is not None and budget <= self.max_budget) + return {"decision": "full_input" if fits else "blocked"} + + +class FakeUsage: + def __init__(self, prompt_tokens, completion_tokens, total_tokens): + self.prompt_tokens = prompt_tokens + self.completion_tokens = completion_tokens + self.total_tokens = total_tokens + + +class FakeChatMessage: + def __init__(self, content, refusal=None): + self.content = content + self.refusal = refusal + + +class FakeChoice: + def __init__(self, message, finish_reason): + self.message = message + self.finish_reason = finish_reason + + +class FakeResponse: + def __init__(self, choices, usage): + self.choices = choices + self.usage = usage + + +COMPOSED_REPLY = "Your daily digest found three new files and one changed report." + + +def fake_response(text=COMPOSED_REPLY, *, finish_reason="stop", refusal=None, usage=(120, 30, 150)): + tokens = FakeUsage(*usage) if usage is not None else None + return FakeResponse([FakeChoice(FakeChatMessage(text, refusal), finish_reason)], tokens) + + +def fake_extract_text(response): + return response.choices[0].message.content + + +class FakeModel: + def __init__(self, deployment, *, provider="aoai", endpoint_id="endpoint-1", model_id=None): + self.deployment = deployment + self.provider = provider + self.behavior_name = deployment + self.model_metadata = None + self.endpoint_id = endpoint_id + self.model_id = model_id or f"model-{deployment}" + self.replies = [] + self.requests = [] + self.closed = 0 + + def metadata(self): + return { + "model_deployment_name": self.deployment, + "model_provider": self.provider, + "model_endpoint_id": self.endpoint_id, + "model_id": self.model_id, + } + + def create_completion(self, messages=None, temperature=None, max_tokens=None): + self.requests.append({"messages": copy.deepcopy(messages), "temperature": temperature, "max_tokens": max_tokens}) + reply = self.replies.pop(0) if self.replies else fake_response() + if isinstance(reply, BaseException): + raise reply + return reply + + def close(self): + self.closed += 1 + + +class FakeModels: + """``resolve_orchestration_model``: the request's selected model, then the default model.""" + + def __init__(self): + self.selected = FakeModel("gpt-selected") + self.default = FakeModel("gpt-default") + self.unavailable = set() + self.calls = [] + + def __call__(self, settings, user_id=None, seeds=None, identity_context=None): + which = "selected" if isinstance(seeds, dict) and seeds.get("model") else "default" + self.calls.append({"which": which, "seeds": copy.deepcopy(seeds), "user_id": user_id, "identity": identity_context}) + if which in self.unavailable: + raise FakeModelError(which) + return self.selected if which == "selected" else self.default + + @property + def requests(self): + return self.selected.requests + self.default.requests + + +class FakeCheck: + def __init__(self, blocked=False, categories=()): + self.blocked = blocked + self.categories = list(categories) + + +class FakeContentChecker: + def __init__(self): + self.calls = [] + self.block = False + self.error = None + + def __call__(self, text, surface, user_id=None, settings=None): + self.calls.append({"text": text, "surface": surface, "user_id": user_id}) + if self.error is not None: + raise self.error + return FakeCheck(blocked=self.block, categories=["hate"] if self.block else []) + + +def fake_attach_chat_check(document, check): + document["content_check"] = { + "blocked": bool(getattr(check, "blocked", False)), + "categories": list(getattr(check, "categories", [])), + } + return document + + +class Recorder: + """A callable that records its calls and raises queued errors first.""" + + def __init__(self, result=None): + self.calls = [] + self.errors = [] + self.result = result + + def __call__(self, *args, **kwargs): + self.calls.append({"args": copy.deepcopy(args), "kwargs": copy.deepcopy(kwargs)}) + if self.errors: + raise self.errors.pop(0) + return self.result + + +class FakeNotifications: + """Both notification helpers. Each idempotency key stores one notice, like the real helpers.""" + + def __init__(self): + self.chat_calls = [] + self.notice_calls = [] + self.stored = {} + self.chat_errors = [] + self.notice_errors = [] + + def chat_response(self, user_id, conversation_id, message_id, conversation_title=None, response_preview=None, + idempotency_key=None, strict=False): + call = { + "type": "chat_response", + "user_id": user_id, + "conversation_id": conversation_id, + "message_id": message_id, + "conversation_title": conversation_title, + "response_preview": response_preview, + "idempotency_key": idempotency_key, + "strict": strict, + } + self.chat_calls.append(call) + if self.chat_errors: + raise self.chat_errors.pop(0) + self.stored.setdefault(idempotency_key, call) + return {"id": f"notice-{len(self.stored)}"} + + def create(self, user_id=None, notification_type=None, title=None, message=None, link_url=None, metadata=None, + idempotency_key=None, strict=False, **kwargs): + call = { + "type": notification_type, + "user_id": user_id, + "title": title, + "message": message, + "link_url": link_url, + "metadata": copy.deepcopy(metadata), + "idempotency_key": idempotency_key, + "strict": strict, + } + self.notice_calls.append(call) + if self.notice_errors: + raise self.notice_errors.pop(0) + self.stored.setdefault(idempotency_key, call) + return {"id": f"notice-{len(self.stored)}"} + + def notices(self, notification_type=None): + return [call for call in self.stored.values() if notification_type is None or call["type"] == notification_type] + + +class FakeLock: + def __init__(self, available=True): + self.available = available + self.acquired = [] + self.released = [] + + def acquire(self, name, seconds): + self.acquired.append((name, seconds)) + return {"id": name, "seconds": seconds} if self.available else None + + def release(self, lock): + self.released.append(lock) + + +class FakeHints: + def __init__(self): + self.queue = [] + self.signals = [] + + def drain(self, limit): + drained = self.queue[:limit] + del self.queue[:limit] + return drained + + def signal(self, user_id, run_id): + self.signals.append((user_id, run_id)) + if (user_id, run_id) not in self.queue: + self.queue.append((user_id, run_id)) + return True + + +class FakeGates: + def __init__(self): + self.workflows = True + self.results = True + self.calls = [] + + def workflows_enabled(self, settings, user_roles=None): + self.calls.append(("workflows", copy.deepcopy(user_roles))) + return self.workflows + + def results_enabled(self, settings, user_roles=None): + self.calls.append(("results", copy.deepcopy(user_roles))) + return self.results + + +class FakeStreamProbe: + def __init__(self): + self.active = False + self.calls = [] + + def __call__(self, user_id, conversation_id, now): + self.calls.append((user_id, conversation_id, now)) + return self.active + + +class FakeMonotonic: + def __init__(self): + self.value = 1000.0 + self.step = 0.0 + + def __call__(self): + self.value += self.step + return self.value + + +WORKFLOW_NAME = "Daily digest" +REQUEST_TEXT = "Run my daily digest and tell me what it found." +SELECTED_MODEL = {"model": {"model_deployment": "gpt-selected"}, "reasoning_effort": "", "active_group_ids": []} +# Mirrors functions_workflow_result_followup.EXCERPT_BUDGET_STEPS; only steps below 48 KiB are used. +EXCERPT_STEPS = (48 * 1024, 24 * 1024, 12 * 1024, 6 * 1024) + + +def make_workflow(**overrides): + workflow = { + "id": WORKFLOW_ID, + "user_id": USER, + "name": WORKFLOW_NAME, + "durable_execution": True, + "definition_revision": "rev-1", + } + workflow.update(overrides) + return workflow + + +def make_conversation(**overrides): + conversation = { + "id": CONVERSATION_ID, + "user_id": USER, + "title": "Planning chat", + "chat_type": "personal", + "last_updated": "2026-05-04T14:05:10.000000", + "has_unread_assistant_response": False, + "last_unread_assistant_message_id": None, + "last_unread_assistant_at": None, + } + conversation.update(overrides) + return conversation + + +def _thread(thread_id): + return {"thread_id": thread_id, "previous_thread_id": None, "active_thread": True, "thread_attempt": 1} + + +def make_request_messages(): + return [ + { + "id": USER_MESSAGE_ID, + "conversation_id": CONVERSATION_ID, + "role": "user", + "content": REQUEST_TEXT, + "timestamp": "2026-05-04T14:05:00.000000", + "metadata": {"thread_info": _thread("thread-request")}, + }, + { + "id": "message-assistant-1", + "conversation_id": CONVERSATION_ID, + "role": "assistant", + "content": "I started Daily digest. I'll post the results here when the run finishes.", + "timestamp": "2026-05-04T14:05:10.000000", + "metadata": {"thread_info": _thread("thread-request")}, + }, + ] + + +def make_invocation(**overrides): + invocation = { + "version": 1, + "source": "chat_orchestration", + "conversation_id": CONVERSATION_ID, + "user_message_id": USER_MESSAGE_ID, + "orchestration_run_id": ORCHESTRATION_RUN_ID, + "attempt_root_run_id": ATTEMPT_ROOT_RUN_ID, + "step_id": STEP_ID, + "requested_by": USER, + "requested_at": REQUESTED_AT, + } + invocation.update(overrides) + return invocation + + +def make_record(*, time_zone="America/New_York", model_selection=None, roles=("User",), deadline_at=None, **overrides): + seed = build_chat_delivery_seed( + time_zone=time_zone, + model_selection=SELECTED_MODEL if model_selection is None else model_selection, + requester_roles=list(roles), + now=parse_iso(REQUESTED_AT), + ) + record = finalize_chat_delivery_seed(seed, {"limits": {"deadline_at": deadline_at}} if deadline_at else None) + record.update(overrides) + return record + + +def make_run(*, status="completed", record=None, **overrides): + run = { + "id": RUN_ID, + "user_id": USER, + "workflow_id": WORKFLOW_ID, + "workflow_name": WORKFLOW_NAME, + "status": status, + "trigger_source": "chat_orchestration", + "definition_revision": "rev-1", + "chat_invocation": make_invocation(), + "chat_delivery": make_record() if record is None else record, + } + run.update(overrides) + return run + + +def newest_message_handler(container, query, params, partition_key): + require("ORDER BY c.timestamp DESC" in query and "TOP 1" in query, f"unexpected message query {query!r}") + conversation_id = params.get("@conversation_id") + require(partition_key == conversation_id, "the newest-message query must stay in the chat's partition") + rows = [item for item in container.items.values() if item.get("conversation_id") == conversation_id] + rows.sort(key=lambda item: str(item.get("timestamp") or ""), reverse=True) + return [ + { + "id": row["id"], + "role": row.get("role"), + "timestamp": row.get("timestamp"), + "thread_info": (row.get("metadata") or {}).get("thread_info"), + } + for row in rows[:1] + ] + + +def _at_or_before(value, moment): + return isinstance(value, str) and value <= moment + + +def sweep_handler(container, query, params, partition_key): + """A Python mirror of the sweep's WHERE clause, so the sweep can be driven end to end.""" + require("IS_DEFINED(c.chat_delivery)" in query, f"unexpected run query {query!r}") + now = params["@now"] + rows = [] + for item in container.items.values(): + record = item.get("chat_delivery") + if not isinstance(record, dict): + continue + next_attempt = record.get("next_attempt_at") + if next_attempt is not None and not _at_or_before(next_attempt, now): + continue + status = record.get("status") + due = ( + status == params["@ready"] + or (status == params["@delivering"] and _at_or_before(record.get("lease_expires_at"), now)) + or ( + status == params["@pending"] + and ( + _at_or_before(record.get("expires_at"), now) + or (item.get("status") in params["@terminal"] and item.get("_ts", 0) <= params["@grace_ts"]) + ) + ) + ) + if due: + rows.append({"id": item["id"], "user_id": item["user_id"]}) + return rows[: params["@top"]] + + +@dataclass +class DeliveryWorld: + services: object + settings: dict + clock: FakeClock + runs: FakeContainer + workflows: FakeContainer + conversations: FakeContainer + messages: FakeContainer + runtime: FakeRuntime + reader: FakeResultReader + disclosure: FakeDisclosure + prompts: FakePromptBuilder + nonces: FakeNonces + budget: FakeBudget + models: FakeModels + checker: FakeContentChecker + notifications: FakeNotifications + tokens: Recorder + incidents: Recorder + cache_bumps: Recorder + identities: Recorder + lock: FakeLock + hints: FakeHints + gates: FakeGates + stream: FakeStreamProbe + monotonic: FakeMonotonic + unread_calls: list = field(default_factory=list) + + def run(self, run_id=RUN_ID): + return self.runs.get(run_id) + + def record(self, run_id=RUN_ID): + return (self.run(run_id) or {}).get("chat_delivery") + + def conversation(self): + return self.conversations.get(CONVERSATION_ID) + + def delivery_messages(self): + return sorted( + (item for item in self.messages.items.values() if str(item["id"]).startswith("assistant_workflow_delivery_")), + key=lambda item: item["timestamp"], + ) + + def set_record(self, run_id=RUN_ID, **changes): + run = self.runs.items[run_id] + record = dict(run["chat_delivery"]) + record.update(changes) + run["chat_delivery"] = record + self.runs.put(run) + return record + + +def make_world(*, run=None, state="completed", version=5, control=True, settings=None): + clock = FakeClock() + world_settings = {"enable_workflows": True, "enable_chat_workflow_results": True} if settings is None else settings + runtime = FakeRuntime() + if control: + runtime.set(state=state, version=version) + unread_calls = [] + + def mark_unread(container, conversation_id, user_id, message_id, planned_at, *, require_private=True, + skip_if_read_since=None): + unread_calls.append({ + "conversation_id": conversation_id, + "user_id": user_id, + "message_id": message_id, + "planned_at": planned_at, + "require_private": require_private, + "skip_if_read_since": skip_if_read_since, + }) + return mark_conversation_unread_guarded( + container, conversation_id, user_id, message_id, planned_at, + require_private=require_private, skip_if_read_since=skip_if_read_since, + ) + + world = DeliveryWorld( + services=None, + settings=world_settings, + clock=clock, + runs=FakeContainer("runs", [make_run() if run is None else run], query_handler=sweep_handler, clock=clock), + workflows=FakeContainer("workflows", [make_workflow()]), + conversations=FakeContainer("conversations", [make_conversation()]), + messages=FakeContainer("messages", make_request_messages(), query_handler=newest_message_handler), + runtime=runtime, + reader=FakeResultReader(), + disclosure=FakeDisclosure(), + prompts=FakePromptBuilder(), + nonces=FakeNonces(), + budget=FakeBudget(), + models=FakeModels(), + checker=FakeContentChecker(), + notifications=FakeNotifications(), + tokens=Recorder(), + incidents=Recorder(), + cache_bumps=Recorder(), + identities=Recorder(result={"identity": "captured"}), + lock=FakeLock(), + hints=FakeHints(), + gates=FakeGates(), + stream=FakeStreamProbe(), + monotonic=FakeMonotonic(), + unread_calls=unread_calls, + ) + world.services = WorkflowChatDeliveryServices( + runs=world.runs, + workflows=world.workflows, + conversations=world.conversations, + messages=world.messages, + get_settings=lambda: dict(world.settings), + runtime_store_factory=runtime.factory, + runtime_conflict_error=FakeRuntimeConflict, + runtime_unavailable_error=FakeRuntimeUnavailable, + read_workflow_result=world.reader, + result_unavailable_error=FakeResultUnavailable, + workflow_result_context=fake_workflow_result_context, + format_run_time=fake_format_run_time, + format_disclosure=world.disclosure, + build_result_messages=world.prompts, + new_fence_nonce=world.nonces, + excerpt_budget_steps=EXCERPT_STEPS, + calculate_budget=world.budget, + resolve_model=world.models, + model_error=FakeModelError, + capture_identity=world.identities, + extract_text=fake_extract_text, + check_content=world.checker, + attach_chat_check=fake_attach_chat_check, + record_incident=world.incidents, + is_message_retracted=lambda message: bool(message.get("retracted")), + create_chat_response_notification=world.notifications.chat_response, + create_notification=world.notifications.create, + log_token_usage=world.tokens, + bump_cache_version=world.cache_bumps, + mark_unread_guarded=mark_unread, + is_conversation_private=conversation_is_private, + is_user_workflows_enabled=world.gates.workflows_enabled, + is_chat_workflow_results_enabled=world.gates.results_enabled, + stream_activity_probe=world.stream, + acquire_lock=world.lock.acquire, + release_lock=world.lock.release, + drain_hints=world.hints.drain, + signal_delivery=world.hints.signal, + clock=clock, + compose_output_tokens=None, + monotonic=world.monotonic, + ) + return world diff --git a/functional_tests/test_workflow_chat_delivery_worker.py b/functional_tests/test_workflow_chat_delivery_worker.py new file mode 100644 index 000000000..69e2a1d2a --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_worker.py @@ -0,0 +1,1642 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_worker.py +""" +Functional test for the workflow chat delivery worker. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures that the worker posts a chat-started workflow run's outcome back into the chat +that asked exactly once per delivery generation, under a dated label, with one unread mark and +one bell notice; that it posts nothing and sends one notice when the chat can't take it; and that +crashes, outages, deferrals, deletions and durable resumes never post twice or retry forever. + +The worker and the delivery contract are real. Storage, the runtime store, the result reader, the +models and the notification helpers are the in-memory fakes in +``test_support/workflow_chat_delivery_fakes.py``. +""" + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +import functions_workflow_chat_delivery_worker as worker # noqa: E402 +from functions_conversation_unread import clear_conversation_unread # noqa: E402 +from functions_workflow_chat_delivery import ( # noqa: E402 + COMPOSE_MAX_TOKENS, + COMPOSE_TEMPERATURE, + EXPIRED_NOTICE_MESSAGE, + FALLBACK_QUESTION, + KIND_ANALYSIS, + KIND_CANCELLED, + KIND_CONTENT_BLOCKED, + KIND_FAILED, + KIND_SKIPPED, + KIND_STATUS, + MAX_ATTEMPTS, + REASON_CONTENT_BLOCKED, + REASON_RESULT_UNAVAILABLE, + REASON_RESULTS_OFF, + SWEEP_LOCK_NAME, + SWEEP_LOCK_SECONDS, + SWEEP_MAX_RUNS, + TRIMMED_FALLBACK_INTRO, + UNDELIVERABLE_NOTICE_MESSAGE, + assemble_delivery_content, + delivery_note_text, + delivery_thread_id, + next_backoff_seconds, + workflow_delivery_message_id, +) +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + COMPOSED_REPLY, + CONVERSATION_ID, + OTHER_USER, + ORCHESTRATION_RUN_ID, + REQUEST_TEXT, + REQUESTED_AT, + RESULT_SHA, + RESULT_TEXT, + RUN_ID, + SELECTED_MODEL, + STEP_ID, + USER, + USER_MESSAGE_ID, + WORKFLOW_ID, + FakeResultUnavailable, + FakeRuntimeUnavailable, + cosmos_error, + fake_response, + make_record, + make_run, + make_world, + parse_iso, + shift, +) + + +LABEL = "Results from `Daily digest` \u00b7 you asked on May 4, 2026 at 10:05 AM (America/New_York)" +DISCLOSURE = "Source: workflow run from May 4, 2026." +M5 = workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) +M8 = workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 8) +NOTICE_LINK = f"/workflow-activity?workflowId={WORKFLOW_ID}&runId={RUN_ID}&scope=personal" +READY_TITLE = 'Results from "Daily digest" are ready' +DELIVERED_PREVIEW = 'Results from "Daily digest" are in your chat' + + +def deliver(world, run_id=RUN_ID, user_id=USER): + return worker.process_workflow_chat_delivery(user_id, run_id, services=world.services) + + +def make_due(world, run_id=RUN_ID): + """Move the clock just past the record's next attempt.""" + world.clock.now = shift(world.record(run_id)["next_attempt_at"], seconds=1) + + +def deliver_until(world, stop_outcomes, *, limit=40): + """Deliver and move to the next attempt until an outcome in ``stop_outcomes`` comes back.""" + outcomes = [] + for _ in range(limit): + outcome = deliver(world) + outcomes.append(outcome) + if outcome in stop_outcomes: + return outcomes + make_due(world) + raise AssertionError(f"no outcome in {stop_outcomes} after {limit} attempts: {outcomes}") + + +def set_run_status(world, status, run_id=RUN_ID): + run = world.run(run_id) + run["status"] = status + world.runs.put(run) + + +def delivered_notice(message_id=M5, generation=5): + return { + "type": "chat_response", + "user_id": USER, + "conversation_id": CONVERSATION_ID, + "message_id": message_id, + "conversation_title": "Planning chat", + "response_preview": DELIVERED_PREVIEW, + "idempotency_key": f"workflow-chat-delivery:{RUN_ID}:{generation}", + "strict": True, + } + + +def undeliverable_notice(title=READY_TITLE, generation=5): + return { + "type": "workflow_chat_delivery", + "user_id": USER, + "title": title, + "message": UNDELIVERABLE_NOTICE_MESSAGE, + "link_url": NOTICE_LINK, + "metadata": { + "workflow_id": WORKFLOW_ID, + "run_id": RUN_ID, + "workflow_scope": "personal", + "delivery_status": "undeliverable", + }, + "idempotency_key": f"workflow-chat-delivery-notice:{RUN_ID}:{generation}", + "strict": True, + } + + +def total_writes(world): + return len(world.runs.writes()) + len(world.messages.writes()) + len(world.conversations.writes()) + + +def note_content(kind, reason_code=None): + return assemble_delivery_content(LABEL, delivery_note_text(kind, "Daily digest", reason_code=reason_code)) + + +def result_contexts(): + return [{"workflow_id": WORKFLOW_ID, "run_id": RUN_ID, "result_sha256": RESULT_SHA}] + + +def terminal_world(state): + """A world whose run ended in ``state`` (the runtime control and the run document agree).""" + return make_world(run=make_run(status=state), state=state) + + +def edit_request_message(world, **changes): + message = world.messages.get(USER_MESSAGE_ID) + message.update(changes) + world.messages.put(message) + + +# --------------------------------------------------------------------------------------------- +# A. The delivered reply +# --------------------------------------------------------------------------------------------- + + +def test_app_version_includes_chat_delivery(): + assert_app_version_at_least("0.261.218") + + +def test_delivers_a_composed_reply_once_with_one_unread_mark_and_one_notice(): + world = make_world() + + outcome = deliver(world) + + assert outcome == worker.OUTCOME_DELIVERED + messages = world.delivery_messages() + assert [message["id"] for message in messages] == [M5] + message = messages[0] + assert message["role"] == "assistant" + assert message["conversation_id"] == CONVERSATION_ID + assert message["content"] == f"{LABEL}\n\n{COMPOSED_REPLY}\n\n{DISCLOSURE}" + assert message["timestamp"] == "2026-05-04T15:00:00.000000" + assert message["model_deployment_name"] == "gpt-selected" + assert message["content_check"] == {"blocked": False, "categories": []} + metadata = message["metadata"] + assert metadata["workflow_delivery"] == { + "version": 1, + "kind": "result", + "workflow_id": WORKFLOW_ID, + "workflow_scope": "personal", + "run_id": RUN_ID, + "generation": 5, + "run_status": "completed", + "orchestration_run_id": ORCHESTRATION_RUN_ID, + "step_id": STEP_ID, + "requested_at": REQUESTED_AT, + } + assert metadata["token_usage"] == {"prompt_tokens": 120, "completion_tokens": 30, "total_tokens": 150} + assert metadata["user_info"] == {"user_id": USER} + assert metadata["thread_info"] == { + "thread_id": delivery_thread_id(M5), + "previous_thread_id": "thread-request", + "active_thread": True, + "thread_attempt": 1, + } + assert metadata["workflow_result"]["result_sha256"] == RESULT_SHA + assert metadata["workflow_result_contexts"] == [ + {"workflow_id": WORKFLOW_ID, "run_id": RUN_ID, "result_sha256": RESULT_SHA}, + ] + + conversation = world.conversation() + assert conversation["has_unread_assistant_response"] is True + assert conversation["last_unread_assistant_message_id"] == M5 + assert conversation["last_unread_assistant_at"] == "2026-05-04T15:00:00.000000" + assert [call["skip_if_read_since"] for call in world.unread_calls] == [None] + assert world.unread_calls[0]["require_private"] is True + assert world.cache_bumps.calls == [{"args": (USER,), "kwargs": {"reason": "workflow_chat_delivery"}}] + + record = world.record() + assert record["status"] == "delivered" + assert record["notice_kind"] == "chat_response" + assert record["phase"] == "notified" + assert record["generation"] == 5 + assert record["attempts"] == 1 + assert record["kind"] == "result" + assert record["run_status"] == "completed" + assert record["message_id"] == M5 + assert record["lease_id"] is None and record["lease_expires_at"] is None + assert record["outcome_reason"] is None + assert record["delivered_at"] == "2026-05-04T15:00:00.000000Z" + + assert world.notifications.chat_calls == [delivered_notice()] + assert world.notifications.notice_calls == [] + + assert [call["which"] for call in world.models.calls] == ["selected"] + assert world.models.calls[0]["seeds"]["model"] == SELECTED_MODEL["model"] + assert world.models.calls[0]["identity"] == {"identity": "captured"} + assert world.identities.calls == [{"args": (USER,), "kwargs": {"conversation_id": CONVERSATION_ID}}] + assert world.models.selected.closed == 1 + assert world.models.default.requests == [] + request = world.models.selected.requests[0] + assert request["temperature"] == COMPOSE_TEMPERATURE + assert request["max_tokens"] == COMPOSE_MAX_TOKENS + assert world.prompts.calls == [ + {"budget": 48 * 1024, "question": REQUEST_TEXT, "nonce": "nonce-1", "time_zone": "America/New_York"}, + ] + assert world.reader.calls[0]["budget"] == 48 * 1024 + assert world.checker.calls == [{"text": message["content"], "surface": "chat_output", "user_id": USER}] + assert world.gates.calls == [("workflows", ["User"]), ("results", ["User"])] + assert world.tokens.calls == [{ + "args": (), + "kwargs": { + "user_id": USER, + "token_type": "chat", + "conversation_id": CONVERSATION_ID, + "message_id": M5, + "model": "gpt-selected", + "workspace_type": "personal", + "total_tokens": 150, + "prompt_tokens": 120, + "completion_tokens": 30, + "additional_context": {"workflow_result_delivery": True}, + "idempotency_key": f"workflow_result_delivery:{M5}:1", + }, + }] + + +def test_reprocessing_a_delivered_run_changes_nothing(): + world = make_world() + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + writes = total_writes(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + + assert total_writes(world) == writes + assert len(world.delivery_messages()) == 1 + assert len(world.notifications.chat_calls) == 1 + assert len(world.unread_calls) == 1 + assert len(world.models.requests) == 1 + + +def test_a_token_log_failure_never_fails_delivery(): + world = make_world() + world.tokens.errors.append(RuntimeError("token log down")) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert len(world.tokens.calls) == 1 + assert world.delivery_messages()[0]["metadata"]["token_usage"]["total_tokens"] == 150 + assert world.record()["notice_kind"] == "chat_response" + + +def test_a_reply_without_usage_logs_no_tokens(): + world = make_world() + world.models.selected.replies.append(fake_response(usage=None)) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.tokens.calls == [] + message = world.delivery_messages()[0] + assert message["content"] == f"{LABEL}\n\n{COMPOSED_REPLY}\n\n{DISCLOSURE}" + assert not message["metadata"].get("token_usage") + + +# --------------------------------------------------------------------------------------------- +# B. What gets posted +# --------------------------------------------------------------------------------------------- + + +def test_steps_the_excerpt_down_bound_to_the_first_digest_until_it_fits(): + world = make_world() + world.budget.max_budget = 12 * 1024 + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [call["budget"] for call in world.reader.calls] == [48 * 1024, 24 * 1024, 12 * 1024], ( + "each smaller excerpt must be read in turn until the prompt fits" + ) + assert [call["expected_sha256"] for call in world.reader.calls] == [None, RESULT_SHA, RESULT_SHA], ( + "a smaller excerpt must be bound to the digest first read" + ) + assert [call["nonce"] for call in world.prompts.calls] == ["nonce-1", "nonce-2", "nonce-3"], ( + "every prompt must carry a fresh fence nonce" + ) + assert len(world.models.requests) == 1, "only the prompt that fits may be sent" + assert world.delivery_messages()[0]["content"] == f"{LABEL}\n\n{COMPOSED_REPLY}\n\n{DISCLOSURE}" + + +def test_a_result_too_large_for_any_excerpt_posts_the_status_note(): + world = make_world() + world.budget.max_budget = 1024 + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == note_content(KIND_STATUS) + assert message["metadata"]["workflow_delivery"]["kind"] == KIND_STATUS + assert "workflow_result" not in message["metadata"] + assert "workflow_result_contexts" not in message["metadata"] + assert [call["which"] for call in world.models.calls] == ["selected"] + assert world.models.requests == [] + assert world.tokens.calls == [] + assert world.record()["outcome_reason"] == REASON_RESULT_UNAVAILABLE + assert world.notifications.chat_calls == [delivered_notice()] + + +def test_uses_the_default_model_when_the_requests_model_is_unavailable(): + world = make_world() + world.models.unavailable.add("selected") + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [call["which"] for call in world.models.calls] == ["selected", "default"] + message = world.delivery_messages()[0] + assert message["model_deployment_name"] == "gpt-default" + assert message["content"] == f"{LABEL}\n\n{COMPOSED_REPLY}\n\n{DISCLOSURE}" + assert world.tokens.calls[0]["kwargs"]["model"] == "gpt-default" + assert world.models.selected.requests == [] + + +def test_uses_the_default_model_when_the_request_named_none(): + world = make_world() + world.set_record(model_selection=None) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [call["which"] for call in world.models.calls] == ["default"] + assert world.delivery_messages()[0]["model_deployment_name"] == "gpt-default" + + +def test_falls_back_to_the_start_of_the_result_when_neither_model_answers(): + world = make_world() + world.models.selected.replies.append(fake_response(finish_reason="content_filter")) + world.models.default.replies.append(fake_response(refusal="I can't help with that.")) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == ( + f"{LABEL}\n\n{TRIMMED_FALLBACK_INTRO}\n\n```\n{RESULT_TEXT}\n```\n\n" + "Source: workflow run from May 4, 2026 (truncated)." + ) + assert message["model_deployment_name"] is None + assert message["metadata"]["workflow_delivery"]["kind"] == "result" + assert message["metadata"]["workflow_result_contexts"] == result_contexts() + assert world.tokens.calls == [], "no compose answered, so there is nothing to count" + assert [call["which"] for call in world.models.calls] == ["selected", "default"] + assert world.models.selected.closed == 1 and world.models.default.closed == 1 + assert len(world.checker.calls) == 1, "the fallback text must be checked like a reply" + + +def test_the_fallback_fence_outgrows_backticks_in_the_result(): + world = make_world() + world.reader.text = "Totals: ```` 3 files ````" + world.models.selected.replies.append(fake_response(finish_reason="content_filter")) + world.models.default.replies.append(fake_response(finish_reason="content_filter")) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert "\n`````\nTotals: ```` 3 files ````\n`````" in world.delivery_messages()[0]["content"] + + +def test_never_asks_the_same_model_binding_twice(): + world = make_world() + world.models.default = world.models.selected + world.models.selected.replies.append(fake_response(finish_reason="content_filter")) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert len(world.models.selected.requests) == 1 + assert world.delivery_messages()[0]["content"].startswith(f"{LABEL}\n\n{TRIMMED_FALLBACK_INTRO}") + + +def test_without_use_workflow_results_in_chat_nothing_of_the_result_is_read(): + world = make_world() + world.gates.results = False + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.reader.calls == [], "the result must not be read when the results gate is off" + assert world.models.calls == [], "no model may see the result when the results gate is off" + message = world.delivery_messages()[0] + assert message["content"] == note_content(KIND_STATUS) + assert "workflow_result" not in message["metadata"] + assert world.record()["outcome_reason"] == REASON_RESULTS_OFF + assert ("results", ["User"]) in world.gates.calls + + +def test_a_saved_analysis_posts_the_analysis_note_with_its_result_context(): + world = make_world() + world.reader.saved_inputs = [{"name": "report.csv"}] + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == note_content(KIND_ANALYSIS) + assert message["metadata"]["workflow_delivery"]["kind"] == KIND_ANALYSIS + assert message["metadata"]["workflow_result_contexts"] == result_contexts() + assert world.models.calls == [] + + +def test_a_closed_result_posts_the_status_note(): + world = make_world() + world.reader.outcomes.append(FakeResultUnavailable("workflow_result_source_access_lost")) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == note_content(KIND_STATUS) + assert "workflow_result" not in message["metadata"] + assert world.models.calls == [] + + +def test_a_blocked_reply_posts_the_blocked_note_and_records_an_incident(): + world = make_world() + world.checker.block = True + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == note_content(KIND_CONTENT_BLOCKED) + assert COMPOSED_REPLY not in message["content"] + assert message["metadata"]["workflow_delivery"]["kind"] == KIND_CONTENT_BLOCKED + assert "workflow_result" not in message["metadata"] + assert message["model_deployment_name"] is None + assert world.record()["outcome_reason"] == REASON_CONTENT_BLOCKED + assert world.incidents.calls == [{ + "args": ({ + "id": M5, + "conversation_id": CONVERSATION_ID, + "content_check": {"blocked": True, "categories": ["hate"]}, + }, USER), + "kwargs": {}, + }] + assert len(world.tokens.calls) == 1, "the compose still ran, so it is still counted" + + +def test_a_blocked_fallback_posts_the_blocked_note(): + world = make_world() + world.checker.block = True + world.models.selected.replies.append(fake_response(finish_reason="content_filter")) + world.models.default.replies.append(fake_response(finish_reason="content_filter")) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == note_content(KIND_CONTENT_BLOCKED), "the trimmed fallback must be checked too" + assert RESULT_TEXT not in message["content"] + assert TRIMMED_FALLBACK_INTRO in world.checker.calls[0]["text"] + + +def test_a_content_check_outage_retries_and_counts_each_compose_once(): + world = make_world() + world.checker.error = RuntimeError("content safety down") + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + + record = world.record() + assert record["status"] == "ready" + assert record["attempts"] == 1 + assert record["next_attempt_at"] == "2026-05-04T15:00:30.000000Z" + assert world.delivery_messages() == [] + assert world.unread_calls == [], "the chat must not be marked unread before the content is ready" + assert world.notifications.chat_calls == [] + + world.checker.error = None + make_due(world) + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.record()["attempts"] == 2 + assert [call["kwargs"]["idempotency_key"] for call in world.tokens.calls] == [ + f"workflow_result_delivery:{M5}:1", + f"workflow_result_delivery:{M5}:2", + ] + assert len(world.delivery_messages()) == 1 + + +@pytest.mark.parametrize( + ("state", "gate_reason_code", "reason_code"), + [ + ("failed", None, "failed"), + ("invalid", None, "invalid"), + ("incomplete", None, "incomplete"), + ("failed", "deadline_exceeded", "deadline_exceeded"), + ], +) +def test_a_failed_run_posts_the_failure_note(state, gate_reason_code, reason_code): + world = terminal_world(state) + if gate_reason_code: + world.runtime.set(state=state, version=5, gate_reason_code=gate_reason_code) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == note_content(KIND_FAILED, reason_code=reason_code) + delivery = message["metadata"]["workflow_delivery"] + assert delivery["kind"] == KIND_FAILED + assert delivery["run_status"] == state + assert "workflow_result" not in message["metadata"] + assert world.reader.calls == [] and world.models.calls == [] + assert world.notifications.chat_calls == [delivered_notice()] + + +@pytest.mark.parametrize(("state", "kind"), [("cancelled", KIND_CANCELLED), ("skipped", KIND_SKIPPED)]) +def test_a_cancelled_or_skipped_run_posts_its_note(state, kind): + world = terminal_world(state) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + message = world.delivery_messages()[0] + assert message["content"] == note_content(kind) + assert message["metadata"]["workflow_delivery"]["kind"] == kind + assert world.reader.calls == [] and world.models.calls == [] + + +@pytest.mark.parametrize("change", ["retracted", "removed", "not_a_user_message"]) +def test_a_missing_or_retracted_request_uses_the_fixed_question(change): + world = make_world() + if change == "retracted": + edit_request_message(world, retracted=True) + elif change == "removed": + world.messages.items.pop(USER_MESSAGE_ID) + else: + edit_request_message(world, role="assistant") + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.prompts.calls[0]["question"] == FALLBACK_QUESTION + + +def test_a_long_request_is_clipped_before_it_reaches_the_model(): + world = make_world() + edit_request_message(world, content="x" * 5000) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.prompts.calls[0]["question"] == "x" * worker.QUESTION_MAX_CHARS + assert worker.QUESTION_MAX_CHARS == 4000 + + +# --------------------------------------------------------------------------------------------- +# C. Phases: a crash or outage at any step never posts twice +# --------------------------------------------------------------------------------------------- + + +def user_reads_chat(world): + """What the mark-read route does: clear the unread state and keep ``last_updated``.""" + conversation = clear_conversation_unread(world.conversation()) + world.conversations.put(conversation) + + +def seconds_until_next_attempt(world): + record = world.record() + return (parse_iso(record["next_attempt_at"]) - parse_iso(world.clock.now)).total_seconds() + + +def test_a_failed_create_retries_without_a_second_unread_mark(): + world = make_world() + world.messages.fail("create_item", cosmos_error(503)) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + assert world.record()["phase"] == "unread_marked" + assert world.delivery_messages() == [] + user_reads_chat(world) + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [message["id"] for message in world.delivery_messages()] == [M5] + assert len(world.unread_calls) == 1, "the retry must not mark the chat unread again" + conversation = world.conversation() + assert conversation["has_unread_assistant_response"] is False, ( + "a chat read between the mark and the create stays read (documented limitation)" + ) + assert conversation["last_updated"] == "2026-05-04T15:00:00.000000" + assert len(world.models.requests) == 2 + assert world.record()["attempts"] == 2 + + +def test_a_failed_unread_mark_retries_with_the_earlier_planned_time(): + world = make_world() + world.conversations.fail("replace_item", cosmos_error(503)) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + assert world.record()["phase"] == "publishing" + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [call["skip_if_read_since"] for call in world.unread_calls] == [None, "2026-05-04T15:00:00.000000"] + assert [call["planned_at"] for call in world.unread_calls] == [ + "2026-05-04T15:00:00.000000", + "2026-05-04T15:00:31.000000", + ] + conversation = world.conversation() + assert conversation["has_unread_assistant_response"] is True + assert conversation["last_unread_assistant_message_id"] == M5 + assert conversation["last_unread_assistant_at"] == "2026-05-04T15:00:31.000000" + assert world.delivery_messages()[0]["timestamp"] == "2026-05-04T15:00:31.000000" + + +def test_a_mark_whose_phase_write_failed_is_not_repeated_after_the_user_reads(): + world = make_world() + world.runs.fail("replace_item", None, None, cosmos_error(503)) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + assert world.record()["phase"] == "publishing" + assert world.conversation()["last_unread_assistant_message_id"] == M5 + user_reads_chat(world) + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert len(world.unread_calls) == 2 + assert world.unread_calls[1]["skip_if_read_since"] == "2026-05-04T15:00:00.000000" + assert world.conversation()["has_unread_assistant_response"] is False + assert len(world.cache_bumps.calls) == 1, "only the mark that landed bumps the chat list" + assert [message["id"] for message in world.delivery_messages()] == [M5] + + +def test_a_message_that_already_exists_is_kept_not_duplicated(): + world = make_world() + world.messages.put({ + "id": M5, + "conversation_id": CONVERSATION_ID, + "role": "assistant", + "content": "posted by an earlier attempt", + "timestamp": "2026-05-04T14:59:59.000000", + }) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + messages = world.delivery_messages() + assert [message["id"] for message in messages] == [M5] + assert messages[0]["content"] == "posted by an earlier attempt" + assert world.record()["phase"] == "notified" + assert world.notifications.chat_calls == [delivered_notice()] + + +def test_a_create_whose_reply_was_lost_is_found_on_the_retry(): + world = make_world() + original_create = world.messages.create_item + + def create_then_fail(body=None, **kwargs): + world.messages.create_item = original_create + original_create(body=body, **kwargs) + raise cosmos_error(503) + + world.messages.create_item = create_then_fail + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + assert world.record()["phase"] == "unread_marked" + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [message["id"] for message in world.delivery_messages()] == [M5] + assert len(world.models.requests) == 1, "a message found on the retry is never composed again" + assert len(world.unread_calls) == 1 + assert world.notifications.chat_calls == [delivered_notice()] + + +def test_a_failed_notice_is_sent_on_the_retry_without_reposting(): + world = make_world() + world.notifications.chat_errors.append(RuntimeError("bell down")) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + assert world.record()["phase"] == "message_created" + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert len(world.notifications.chat_calls) == 2 + assert list(world.notifications.stored) == [f"workflow-chat-delivery:{RUN_ID}:5"] + assert len(world.reader.calls) == 1 + assert len(world.unread_calls) == 1 + assert len(world.delivery_messages()) == 1 + assert world.record()["notice_kind"] == "chat_response" + + +def test_a_notice_that_never_lands_backs_off_then_closes_delivered_without_one(): + world = make_world() + world.notifications.chat_errors.extend(RuntimeError("bell down") for _ in range(20)) + + outcomes, waits = [], [] + for _ in range(MAX_ATTEMPTS): + outcome = deliver(world) + outcomes.append(outcome) + if outcome != worker.OUTCOME_RETRY_SCHEDULED: + break + waits.append(seconds_until_next_attempt(world)) + make_due(world) + + assert outcomes == [worker.OUTCOME_RETRY_SCHEDULED] * 7 + [worker.OUTCOME_DELIVERED] + assert waits == [next_backoff_seconds(attempt) for attempt in range(1, 8)] + assert waits == [30, 60, 120, 240, 480, 900, 900] + record = world.record() + assert record["status"] == "delivered" and record["notice_kind"] == "none" + assert record["attempts"] == MAX_ATTEMPTS + assert len(world.notifications.chat_calls) == 9, "eight attempts plus one last best-effort notice" + assert world.notifications.stored == {} + assert len(world.delivery_messages()) == 1 + + +def test_a_created_message_still_gets_its_notice_after_the_window_closes(): + world = make_world() + world.notifications.chat_errors.append(RuntimeError("bell down")) + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + world.clock.now = shift(world.record()["expires_at"], days=1) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.record()["notice_kind"] == "chat_response" + assert len(world.notifications.stored) == 1 + assert world.notifications.notice_calls == [] + + +def test_a_lost_lease_stops_the_attempt_before_the_chat_changes(): + world = make_world() + check = world.services.check_content + + def check_then_lose_lease(*args, **kwargs): + world.set_record(lease_id="lease-other", lease_expires_at="2026-05-04T15:05:00.000000Z") + return check(*args, **kwargs) + + world.services.check_content = check_then_lose_lease + + outcome = deliver(world) + assert outcome == worker.OUTCOME_LOST_LEASE + + assert world.unread_calls == [] + assert world.delivery_messages() == [] + assert world.notifications.chat_calls == [] and world.notifications.notice_calls == [] + assert world.record()["lease_id"] == "lease-other" + + +# --------------------------------------------------------------------------------------------- +# D. Closures: nothing posted, at most one notice +# --------------------------------------------------------------------------------------------- + + +EXPIRED_TITLE = "\"Daily digest\" didn't finish in time to post to chat" + + +def expired_notice(): + notice = undeliverable_notice(title=EXPIRED_TITLE) + notice.update({ + "message": EXPIRED_NOTICE_MESSAGE, + "idempotency_key": f"workflow-chat-delivery-notice:{RUN_ID}:expired", + }) + notice["metadata"]["delivery_status"] = "expired" + return notice + + +def assert_nothing_posted(world): + assert world.delivery_messages() == [] + assert world.unread_calls == [] + assert world.notifications.chat_calls == [] + + +def assert_closed(world, *, status="undeliverable", reason, notice_kind, attempts): + record = world.record() + assert record["status"] == status + assert record["outcome_reason"] == reason + assert record["notice_kind"] == notice_kind + assert record["attempts"] == attempts + assert record["lease_id"] is None and record["next_attempt_at"] is None + + +@pytest.mark.parametrize( + "change", + ["removed", "another_owner", "converted_to_collaboration", "shared", "deleted", "orchestration_deleted"], +) +def test_a_chat_that_cannot_take_the_delivery_gets_one_notice_instead(change): + world = make_world() + if change == "removed": + world.conversations.items.pop(CONVERSATION_ID) + else: + conversation = world.conversation() + conversation.update({ + "another_owner": {"user_id": OTHER_USER}, + "converted_to_collaboration": {"converted_to_collaboration_at": "2026-05-04T14:30:00Z"}, + "shared": {"collaboration_conversation_id": "collaboration-1"}, + "deleted": {"deleted": True}, + "orchestration_deleted": {"orchestration_deleted": True}, + }[change]) + world.conversations.put(conversation) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_UNDELIVERABLE + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [undeliverable_notice()] + assert world.reader.calls == [] and world.models.calls == [] + assert_closed(world, reason="chat_unavailable", notice_kind="undeliverable", attempts=1) + + +def test_losing_workflow_access_gets_one_notice_instead(): + world = make_world() + world.gates.workflows = False + + outcome = deliver(world) + assert outcome == worker.OUTCOME_UNDELIVERABLE + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [undeliverable_notice()] + assert world.gates.calls == [("workflows", ["User"])] + assert_closed(world, reason="access_lost", notice_kind="undeliverable", attempts=1) + + +@pytest.mark.parametrize("change", ["removed", "deleting", "status_deleting", "another_owner"]) +def test_a_deleted_workflow_closes_silently(change): + world = make_world() + if change == "removed": + world.workflows.items.pop(WORKFLOW_ID) + else: + workflow = world.workflows.get(WORKFLOW_ID) + workflow.update({ + "deleting": {"deleting": True}, + "status_deleting": {"status": "deleting"}, + "another_owner": {"user_id": OTHER_USER}, + }[change]) + world.workflows.put(workflow) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_CLOSED_SILENTLY + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [] + assert_closed(world, reason="workflow_deleted", notice_kind="none", attempts=1) + + +def test_a_run_deleted_while_pending_closes_silently_without_an_attempt(): + world = make_world(run=make_run(status="running"), state="running", version=3) + outcome = deliver(world) + assert outcome == worker.OUTCOME_PENDING + world.runtime.tombstone() + + outcome = deliver(world) + assert outcome == worker.OUTCOME_CLOSED_SILENTLY + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [] + assert_closed(world, reason="workflow_deleted", notice_kind="none", attempts=0) + + +def test_a_run_deleted_while_ready_closes_silently_without_an_attempt(): + world = make_world() + world.stream.active = True + outcome = deliver(world) + assert outcome == worker.OUTCOME_DEFERRED + world.runtime.tombstone() + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_CLOSED_SILENTLY + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [] + assert_closed(world, reason="workflow_deleted", notice_kind="none", attempts=0) + + +@pytest.mark.parametrize("change", ["tombstone", "remove"]) +def test_a_run_deleted_after_delivery_stays_delivered(change): + world = make_world() + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + getattr(world.runtime, change)() + writes = total_writes(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + + assert total_writes(world) == writes + assert len(world.delivery_messages()) == 1 + assert len(world.notifications.chat_calls) == 1 + assert world.notifications.notice_calls == [] + assert world.record()["status"] == "delivered" + + +def test_a_missing_runtime_control_closes_silently_without_an_attempt(): + world = make_world(control=False) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_CLOSED_SILENTLY + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [] + assert_closed(world, reason="runtime_missing", notice_kind="none", attempts=0) + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + + +def test_a_run_that_never_finishes_in_the_window_gets_the_expired_notice(): + world = make_world(run=make_run(status="running"), state="running", version=3) + world.clock.now = shift(world.record()["expires_at"], seconds=1) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_EXPIRED + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [expired_notice()] + record = world.record() + assert record["status"] == "expired" and record["notice_kind"] == "expired" + assert record["kind"] == "expired" and record["generation"] == 3 + + world.runtime.update(state="completed", version=5) + set_run_status(world, "completed") + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_APPLICABLE, "an expired record is never reopened" + assert world.delivery_messages() == [] + assert len(world.notifications.notice_calls) == 1 + + +def test_a_run_paused_at_its_deadline_gets_the_expired_notice_at_once(): + world = make_world(run=make_run(status="paused"), state="paused", version=4) + world.runtime.set(state="paused", version=4, gate_reason_code="deadline_exceeded") + + outcome = deliver(world) + assert outcome == worker.OUTCOME_EXPIRED + + assert world.notifications.notice_calls == [expired_notice()] + assert world.delivery_messages() == [] + + +def test_an_expired_notice_that_keeps_failing_still_closes_as_expired(): + world = make_world(run=make_run(status="running"), state="running", version=3) + world.clock.now = shift(world.record()["expires_at"], seconds=1) + world.notifications.notice_errors.extend(RuntimeError("bell down") for _ in range(20)) + + outcomes = deliver_until(world, {worker.OUTCOME_EXPIRED}) + + assert outcomes == [worker.OUTCOME_RETRY_SCHEDULED] * 7 + [worker.OUTCOME_EXPIRED] + record = world.record() + assert record["status"] == "expired" and record["notice_kind"] == "none" + assert world.notifications.stored == {} + + +def test_a_run_that_finished_after_the_window_gets_one_notice_and_no_post(): + world = make_world() + world.clock.now = shift(world.record()["expires_at"], seconds=1) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_UNDELIVERABLE + + assert_nothing_posted(world) + assert world.notifications.notice_calls == [undeliverable_notice()] + assert world.reader.calls == [] + assert_closed(world, reason="expired_before_delivery", notice_kind="undeliverable", attempts=1) + + +def test_a_notice_sent_before_a_failed_close_is_not_sent_twice(): + world = make_world() + world.conversations.items.pop(CONVERSATION_ID) + world.runs.fail("replace_item", None, cosmos_error(503)) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + make_due(world) + outcome = deliver(world) + assert outcome == worker.OUTCOME_UNDELIVERABLE + + assert len(world.notifications.notice_calls) == 2 + assert world.notifications.notices() == [undeliverable_notice()] + assert_closed(world, reason="chat_unavailable", notice_kind="undeliverable", attempts=2) + + +def test_an_undeliverable_notice_that_never_lands_closes_without_one(): + world = make_world() + world.conversations.items.pop(CONVERSATION_ID) + world.notifications.notice_errors.extend(RuntimeError("bell down") for _ in range(20)) + + outcomes = deliver_until(world, {worker.OUTCOME_UNDELIVERABLE}) + + assert outcomes == [worker.OUTCOME_RETRY_SCHEDULED] * 7 + [worker.OUTCOME_UNDELIVERABLE] + assert_closed(world, reason="delivery_failed", notice_kind="none", attempts=MAX_ATTEMPTS) + assert world.notifications.stored == {} + + +# --------------------------------------------------------------------------------------------- +# E. Deferral while the chat is busy (D7) +# --------------------------------------------------------------------------------------------- + + +def add_user_message(world, timestamp, message_id="message-user-2"): + world.messages.put({ + "id": message_id, + "conversation_id": CONVERSATION_ID, + "role": "user", + "content": "Something else while the run worked.", + "timestamp": timestamp, + "metadata": {"thread_info": {"thread_id": "thread-later", "previous_thread_id": "thread-request", + "active_thread": True, "thread_attempt": 1}}, + }) + + +def test_an_active_stream_defers_without_spending_an_attempt(): + world = make_world() + world.stream.active = True + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DEFERRED + + record = world.record() + assert record["status"] == "ready" and record["attempts"] == 0 + assert record["first_deferred_at"] == "2026-05-04T15:00:00.000000Z" + assert seconds_until_next_attempt(world) == 30 + assert world.reader.calls == [] and world.unread_calls == [] + assert world.stream.calls[0][:2] == (USER, CONVERSATION_ID) + world.stream.active = False + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.record()["attempts"] == 1 + + +def test_a_recent_user_message_defers_until_the_chat_has_been_quiet_for_ten_minutes(): + world = make_world() + add_user_message(world, "2026-05-04T14:59:00.000000") + + outcomes = deliver_until(world, {worker.OUTCOME_DELIVERED}) + + assert outcomes == [worker.OUTCOME_DEFERRED] * 18 + [worker.OUTCOME_DELIVERED] + message = world.delivery_messages()[0] + assert message["timestamp"] == "2026-05-04T15:09:18.000000" + assert message["metadata"]["thread_info"]["previous_thread_id"] == "thread-later" + assert world.record()["attempts"] == 1 + + +def test_an_older_user_message_does_not_defer(): + world = make_world() + add_user_message(world, "2026-05-04T14:49:59.000000") + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + +def test_a_chat_that_stays_busy_gets_the_delivery_after_fifteen_minutes(): + world = make_world() + world.stream.active = True + + outcomes = deliver_until(world, {worker.OUTCOME_DELIVERED}) + + assert outcomes == [worker.OUTCOME_DEFERRED] * 30 + [worker.OUTCOME_DELIVERED] + assert world.delivery_messages()[0]["timestamp"] == "2026-05-04T15:15:30.000000" + assert world.record()["attempts"] == 1 + + +def test_unreadable_chat_activity_backs_off_then_gives_up_with_one_notice(): + world = make_world() + world.messages.fail("query_items", *[cosmos_error(503) for _ in range(20)]) + + outcomes, waits = [], [] + for _ in range(MAX_ATTEMPTS): + outcome = deliver(world) + outcomes.append(outcome) + if outcome != worker.OUTCOME_DEFERRED: + break + waits.append(seconds_until_next_attempt(world)) + make_due(world) + + assert outcomes == [worker.OUTCOME_DEFERRED] * 7 + [worker.OUTCOME_UNDELIVERABLE] + assert waits == [30, 60, 120, 240, 480, 900, 900], "a counted deferral backs off like a failed attempt" + assert_nothing_posted(world) + assert world.reader.calls == [] + assert world.notifications.notices() == [undeliverable_notice()] + assert_closed(world, reason="delivery_failed", notice_kind="undeliverable", attempts=MAX_ATTEMPTS) + + +def test_the_message_found_check_comes_before_the_busy_check(): + world = make_world() + world.messages.fail("create_item", cosmos_error(503)) + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + world.stream.active = True + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED, "a marked chat is published even mid-stream" + + assert [message["id"] for message in world.delivery_messages()] == [M5] + + +# --------------------------------------------------------------------------------------------- +# F. Transient failures +# --------------------------------------------------------------------------------------------- + + +def run_until_settled(world, retry_outcome, final_outcome): + outcomes, waits = [], [] + for _ in range(MAX_ATTEMPTS + 2): + outcome = deliver(world) + outcomes.append(outcome) + if outcome != retry_outcome: + break + waits.append(seconds_until_next_attempt(world)) + make_due(world) + assert outcomes[-1] == final_outcome, outcomes + return outcomes, waits + + +def test_an_unreadable_runtime_control_backs_off_then_closes_silently(): + world = make_world() + world.runtime.fail(RUN_ID, *[FakeRuntimeUnavailable() for _ in range(20)]) + + outcomes, waits = run_until_settled(world, worker.OUTCOME_RETRY_SCHEDULED, worker.OUTCOME_CLOSED_SILENTLY) + + assert outcomes == [worker.OUTCOME_RETRY_SCHEDULED] * 7 + [worker.OUTCOME_CLOSED_SILENTLY] + assert waits == [30, 60, 120, 240, 480, 900, 900] + assert_nothing_posted(world) + assert world.notifications.notice_calls == [] + assert_closed(world, reason="delivery_failed", notice_kind="none", attempts=MAX_ATTEMPTS) + assert world.record()["generation"] is None, "no outcome was ever read" + + +def test_an_early_hint_never_spends_an_attempt_on_an_unreadable_control(): + world = make_world() + world.runtime.fail(RUN_ID, FakeRuntimeUnavailable()) + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + world.runtime.fail(RUN_ID, FakeRuntimeUnavailable()) + writes = total_writes(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_DUE + + assert total_writes(world) == writes + assert world.record()["attempts"] == 1 + make_due(world) + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + assert world.record()["attempts"] == 2 + + +def test_a_control_that_recovers_delivers_normally(): + world = make_world() + world.runtime.fail(RUN_ID, FakeRuntimeUnavailable(), FakeRuntimeUnavailable()) + + outcomes, _waits = run_until_settled(world, worker.OUTCOME_RETRY_SCHEDULED, worker.OUTCOME_DELIVERED) + + assert outcomes == [worker.OUTCOME_RETRY_SCHEDULED] * 2 + [worker.OUTCOME_DELIVERED] + assert world.record()["attempts"] == 3 + assert [message["id"] for message in world.delivery_messages()] == [M5] + + +def test_unreadable_result_storage_backs_off_then_gives_up_with_one_notice(): + world = make_world() + world.reader.outcomes.extend(FakeResultUnavailable("workflow_result_storage_unavailable") for _ in range(20)) + + outcomes, waits = run_until_settled(world, worker.OUTCOME_RETRY_SCHEDULED, worker.OUTCOME_UNDELIVERABLE) + + assert outcomes == [worker.OUTCOME_RETRY_SCHEDULED] * 7 + [worker.OUTCOME_UNDELIVERABLE] + assert waits == [30, 60, 120, 240, 480, 900, 900] + assert len(world.reader.calls) == MAX_ATTEMPTS, "every attempt reads the result once" + assert_nothing_posted(world) + assert world.notifications.notices() == [undeliverable_notice()] + assert_closed(world, reason="delivery_failed", notice_kind="undeliverable", attempts=MAX_ATTEMPTS) + + +def test_a_settings_read_failure_retries(): + world = make_world() + calls = {"count": 0} + read_settings = world.services.get_settings + + def flaky_settings(): + calls["count"] += 1 + if calls["count"] == 1: + raise RuntimeError("settings store down") + return read_settings() + + world.services.get_settings = flaky_settings + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + assert world.record()["phase"] == "claimed" + make_due(world) + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + +def test_an_unexpected_error_is_retried_like_a_failed_attempt(): + world = make_world() + + def broken_probe(*args, **kwargs): + raise ValueError("unexpected") + + world.services.stream_activity_probe = broken_probe + + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + + record = world.record() + assert record["status"] == "ready" and record["lease_id"] is None + assert seconds_until_next_attempt(world) == 30 + assert world.delivery_messages() == [] and world.unread_calls == [] + world.services.stream_activity_probe = world.stream + make_due(world) + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + assert world.record()["attempts"] == 2 + + +def test_an_outcome_that_cannot_be_recorded_waits_for_the_lease_to_lapse(): + world = make_world() + world.stream.active = True + world.runs.fail("replace_item", None, cosmos_error(503)) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_UNAVAILABLE + + record = world.record() + assert record["status"] == "delivering" and record["lease_id"] + world.stream.active = False + outcome = deliver(world) + assert outcome == worker.OUTCOME_BUSY + world.clock.now = shift(record["lease_expires_at"], seconds=1) + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + assert world.record()["attempts"] == 2 + + +# --------------------------------------------------------------------------------------------- +# G. Generations: a durable resume is a new outcome +# --------------------------------------------------------------------------------------------- + + +def resume_run(world, version, state="queued"): + """What ``runtime/resume`` does to the control and the run document projection.""" + world.runtime.update(state=state, version=version) + set_run_status(world, state) + + +def test_a_resumed_run_delivers_its_new_outcome_after_the_failed_note(): + world = terminal_world("failed") + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + assert world.delivery_messages()[0]["content"] == note_content(KIND_FAILED) + resume_run(world, 6) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_PENDING + + record = world.record() + assert record["status"] == "pending" and record["generation"] is None and record["attempts"] == 0 + assert record["history"] == [{ + "generation": 5, + "status": "delivered", + "kind": "failed", + "outcome_reason": None, + "message_id": M5, + "delivered_at": "2026-05-04T15:00:00.000000Z", + "notice_kind": "chat_response", + }] + resume_run(world, 8, state="completed") + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [message["id"] for message in world.delivery_messages()] == [M5, M8] + assert world.notifications.chat_calls == [delivered_notice(M5, 5), delivered_notice(M8, 8)] + record = world.record() + assert record["generation"] == 8 and record["kind"] == "result" and record["message_id"] == M8 + assert world.delivery_messages()[1]["metadata"]["workflow_delivery"]["generation"] == 8 + + +def test_a_new_outcome_replaces_a_generation_that_was_never_posted(): + world = make_world() + world.stream.active = True + outcome = deliver(world) + assert outcome == worker.OUTCOME_DEFERRED + world.runtime.update(state="completed", version=8) + world.stream.active = False + make_due(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert [message["id"] for message in world.delivery_messages()] == [M8] + assert world.notifications.chat_calls == [delivered_notice(M8, 8)] + assert world.record()["history"] == [], "a generation that was never posted leaves no history" + + +def test_a_resume_that_finishes_during_a_delivery_is_delivered_next(): + world = make_world() + notify = world.services.create_chat_response_notification + + def notify_then_resume(*args, **kwargs): + world.services.create_chat_response_notification = notify + world.runtime.update(state="completed", version=8) + return notify(*args, **kwargs) + + world.services.create_chat_response_notification = notify_then_resume + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + assert world.hints.signals == [(USER, RUN_ID)] + record = world.record() + assert record["status"] == "ready" and record["generation"] == 8 + assert record["history"][-1]["generation"] == 5 + + outcomes = worker.process_workflow_chat_delivery_hints(world.services) + + assert outcomes == [(RUN_ID, worker.OUTCOME_DELIVERED)] + assert [message["id"] for message in world.delivery_messages()] == [M5, M8] + assert world.hints.queue == [] + + +def test_a_cancelled_resume_does_not_reopen_a_delivered_record(): + world = make_world() + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + world.runtime.update(state="completed", version=5) + writes = total_writes(world) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + + assert total_writes(world) == writes + assert world.record()["history"] == [] + + +# --------------------------------------------------------------------------------------------- +# H. The sweep and the hints (D8) +# --------------------------------------------------------------------------------------------- + + +def add_runs(world, count): + """``count`` finished runs in all, each with its own runtime control.""" + run_ids = [RUN_ID] + for index in range(2, count + 1): + run_id = f"{RUN_ID}-{index}" + world.runs.put(make_run(id=run_id)) + world.runtime.set(run_id=run_id) + run_ids.append(run_id) + return run_ids + + +def sweep(world): + return worker.run_workflow_chat_delivery_sweep(world.services) + + +def test_the_sweep_queries_nothing_without_the_lock(): + world = make_world() + world.lock.available = False + world.clock.advance(minutes=3) + + summary = sweep(world) + assert summary == {"locked": False, "processed": 0, "outcomes": {}} + + assert world.runs.query_calls == [] + assert world.lock.released == [] + assert world.record()["status"] == "pending" + + +def test_the_sweep_leaves_a_just_finished_run_to_its_hint(): + world = make_world() + + summary = sweep(world) + assert summary == {"locked": True, "processed": 0, "outcomes": {}} + world.clock.advance(minutes=3) + + summary = sweep(world) + assert summary == {"locked": True, "processed": 1, "outcomes": {worker.OUTCOME_DELIVERED: 1}} + + assert [message["id"] for message in world.delivery_messages()] == [M5] + + +def test_the_sweep_queries_across_users_under_its_lock(): + world = make_world() + world.clock.advance(minutes=3) + + sweep(world) + + assert world.lock.acquired == [(SWEEP_LOCK_NAME, SWEEP_LOCK_SECONDS)] + assert world.lock.released == [{"id": SWEEP_LOCK_NAME, "seconds": SWEEP_LOCK_SECONDS}] + (call,) = world.runs.query_calls + assert call["query"] == worker.SWEEP_QUERY + assert call["enable_cross_partition_query"] is True and call["partition_key"] is None + params = {entry["name"]: entry["value"] for entry in call["parameters"]} + assert params["@now"] == "2026-05-04T15:03:00.000000Z" + assert params["@grace_ts"] == int(parse_iso("2026-05-04T15:01:00+00:00").timestamp()) + + +def test_a_failed_sweep_query_still_releases_the_lock(): + world = make_world() + world.runs.fail("query_items", cosmos_error(503)) + + summary = sweep(world) + assert summary == {"locked": True, "processed": 0, "outcomes": {}} + + assert len(world.lock.released) == 1 + + +@pytest.mark.parametrize("step, processed", [(40.0, 2), (50.0, 1)]) +def test_the_sweep_stops_at_its_time_budget(step, processed): + world = make_world() + add_runs(world, 5) + world.clock.advance(minutes=3) + world.monotonic.step = step + + summary = sweep(world) + assert summary["processed"] == processed + + +def test_the_sweep_processes_at_most_eight_runs_a_pass(): + world = make_world() + run_ids = add_runs(world, 10) + world.clock.advance(minutes=3) + + summary = sweep(world) + assert summary == { + "locked": True, + "processed": SWEEP_MAX_RUNS, + "outcomes": {worker.OUTCOME_DELIVERED: SWEEP_MAX_RUNS}, + } + summary = sweep(world) + assert summary["processed"] == 2 + + assert [world.record(run_id)["status"] for run_id in run_ids] == ["delivered"] * 10 + assert len(world.delivery_messages()) == 10 + + +def test_the_sweep_closes_a_run_without_a_runtime_control_in_one_pass(): + world = make_world(control=False) + world.clock.advance(minutes=3) + + summary = sweep(world) + assert summary["outcomes"] == {worker.OUTCOME_CLOSED_SILENTLY: 1} + + assert world.record()["outcome_reason"] == "runtime_missing" + summary = sweep(world) + assert summary["processed"] == 0 + assert world.notifications.notice_calls == [] + + +def test_the_sweep_picks_up_a_retry_when_it_is_due(): + world = make_world() + world.messages.fail("create_item", cosmos_error(503)) + outcome = deliver(world) + assert outcome == worker.OUTCOME_RETRY_SCHEDULED + + summary = sweep(world) + assert summary["processed"] == 0 + make_due(world) + summary = sweep(world) + assert summary["outcomes"] == {worker.OUTCOME_DELIVERED: 1} + + +def test_hints_are_processed_without_the_sweep_lock(): + world = make_world() + world.lock.available = False + world.hints.signal(USER, RUN_ID) + + hint_outcomes = worker.process_workflow_chat_delivery_hints(world.services) + assert hint_outcomes == [(RUN_ID, worker.OUTCOME_DELIVERED)] + + assert world.hints.queue == [] + assert world.lock.acquired == [] + + +# --------------------------------------------------------------------------------------------- +# I. Guards +# --------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("user_id, run_id", [("", RUN_ID), (None, RUN_ID), (USER, ""), (USER, None)]) +def test_missing_ids_are_not_applicable(user_id, run_id): + world = make_world() + + outcome = worker.process_workflow_chat_delivery(user_id, run_id, services=world.services) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + + assert total_writes(world) == 0 + + +def test_a_run_without_a_delivery_record_is_not_applicable(): + run = make_run() + run.pop("chat_delivery") + world = make_world(run=run) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + assert total_writes(world) == 0 + + +@pytest.mark.parametrize("user_id, run_id", [(USER, "run-missing"), (OTHER_USER, RUN_ID)]) +def test_a_missing_or_foreign_run_is_not_applicable(user_id, run_id): + world = make_world() + + outcome = deliver(world, run_id=run_id, user_id=user_id) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + + assert total_writes(world) == 0 + assert world.runtime.reads == [] + + +def claimed_record(**changes): + record = make_record(status="delivering", generation=5, kind="result", run_status="completed", phase="claimed", + attempts=1, lease_id="lease-other") + record.update(changes) + return record + + +def test_a_live_lease_is_left_alone(): + world = make_world(run=make_run(record=claimed_record(lease_expires_at="2026-05-04T15:04:00.000000Z"))) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_BUSY + + assert total_writes(world) == 0 + assert world.runtime.reads == [] + + +def test_an_abandoned_claim_is_taken_over_after_its_lease_lapses(): + world = make_world(run=make_run(record=claimed_record(lease_expires_at="2026-05-04T14:59:59.000000Z"))) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + + record = world.record() + assert record["attempts"] == 2 and record["lease_id"] is None + assert [message["id"] for message in world.delivery_messages()] == [M5] + + +def test_a_record_that_is_not_due_is_left_alone(): + record = make_record(status="ready", generation=5, kind="result", run_status="completed", attempts=1, + next_attempt_at="2026-05-04T15:01:00.000000Z") + world = make_world(run=make_run(record=record)) + + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_DUE + + assert total_writes(world) == 0 + + +def test_a_closed_record_is_never_claimed_again(): + world = make_world() + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED + writes = total_writes(world) + + for _ in range(3): + outcome = deliver(world) + assert outcome == worker.OUTCOME_NOT_APPLICABLE + + assert total_writes(world) == writes + assert len(world.notifications.chat_calls) == 1 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From 66cf5b91b8368223b7cf9e389ac0319b0d781cd6 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 01:44:09 -0400 Subject: [PATCH 07/36] Add the workflow chat delivery loop (6b-1) Runs the hint-driven delivery worker in a background thread with a throttled, lock-guarded sweep. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/background_tasks.py | 40 +++ .../test_workflow_chat_delivery_loop.py | 337 ++++++++++++++++++ 2 files changed, 377 insertions(+) create mode 100644 functional_tests/test_workflow_chat_delivery_loop.py diff --git a/application/single_app/background_tasks.py b/application/single_app/background_tasks.py index b4fdf5228..bc4c64c57 100644 --- a/application/single_app/background_tasks.py +++ b/application/single_app/background_tasks.py @@ -921,6 +921,45 @@ def run_orchestration_scheduler_loop(app=None): time.sleep(30) +def run_workflow_chat_delivery_loop(app=None): + """Post chat-started workflow outcomes back into the chats that asked for them. + + A hint from the runtime projection wakes the loop at once. The cross-user sweep runs at most + once per sweep interval, and the worker runs it only while it holds the distributed lock. + """ + # Deferred like the orchestration loop: the worker binds application storage on first use. + import functions_workflow_chat_delivery as delivery_contract + import functions_workflow_chat_delivery_worker as delivery_worker + + interval = delivery_contract.SWEEP_INTERVAL_SECONDS + services = None + next_sweep_at = 0.0 + while True: + started = time.monotonic() + sweep_due = started >= next_sweep_at + if sweep_due: + next_sweep_at = started + interval + try: + if services is None: + services = delivery_worker.default_workflow_chat_delivery_services( + acquire_lock=acquire_distributed_task_lock, + release_lock=release_distributed_task_lock, + ) + with app.app_context() if app is not None else nullcontext(): + delivery_worker.process_workflow_chat_delivery_hints(services) + if sweep_due: + delivery_worker.run_workflow_chat_delivery_sweep(services) + except Exception as exc: + log_event( + '[WORKFLOW_CHAT_DELIVERY] Delivery slice could not complete.', + level=logging.ERROR, extra={'error_type': type(exc).__name__}, + ) + # A failure can leave hints queued, so wait without them to avoid a busy loop. + time.sleep(interval) + continue + delivery_contract.wait_for_workflow_chat_delivery_hint(max(1.0, next_sweep_at - time.monotonic())) + + def run_data_management_scheduler_loop(app=None): """Queue due backup and recoverable migration jobs across scaled-out workers.""" while True: @@ -1045,6 +1084,7 @@ def start_background_task_threads(app=None): ('File Sync scheduler background task started.', run_file_sync_scheduler_loop), ('Tabular generated-output scheduler background task started.', run_tabular_generated_output_scheduler_loop), ('Orchestration scheduler background task started.', lambda: run_orchestration_scheduler_loop(app=app)), + ('Workflow chat delivery background task started.', lambda: run_workflow_chat_delivery_loop(app=app)), ('Data Management scheduler background task started.', lambda: run_data_management_scheduler_loop(app=app)), ('Content screening scheduler background task started.', lambda: run_content_screening_scheduler_loop(app=app)), ('App maintenance background task started.', run_app_maintenance_loop), diff --git a/functional_tests/test_workflow_chat_delivery_loop.py b/functional_tests/test_workflow_chat_delivery_loop.py new file mode 100644 index 000000000..b055dcb2b --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_loop.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_loop.py +""" +Functional test for the workflow chat delivery background loop. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures that the delivery loop binds the worker's services once with the distributed +lock helpers, processes hints on every wake, runs the cross-user sweep at most once per sweep +interval, recovers from failures without a busy loop and logs only the error type, and that +start_background_task_threads registers it with the Flask app. It also drives the real sweep +through the real lock helpers over a fake settings container. +""" + +import ast +import logging +import os +import socket +import sys +import threading +import types +import uuid +from contextlib import contextmanager, nullcontext +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = ROOT / "application" / "single_app" +for _path in (APP_ROOT, ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 +from azure.core import MatchConditions # noqa: E402 + +import functions_workflow_chat_delivery as delivery_contract # noqa: E402 +import functions_workflow_chat_delivery_worker as delivery_worker # noqa: E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import FakeContainer, make_world # noqa: E402 + + +BACKGROUND_TASKS = APP_ROOT / "background_tasks.py" +LOOP_NAME = "run_workflow_chat_delivery_loop" +LOG_MESSAGE = "[WORKFLOW_CHAT_DELIVERY] Delivery slice could not complete." +CANARY = "private-delivery-canary" +INTERVAL = delivery_contract.SWEEP_INTERVAL_SECONDS + + +class LoopStopped(Exception): + """Raised by the fakes to end the endless loop after the scripted wakes.""" + + +def _module_functions(): + parsed = ast.parse(BACKGROUND_TASKS.read_text(encoding="utf-8")) + return {node.name: node for node in parsed.body if isinstance(node, ast.FunctionDef)} + + +def load_background_functions(names, namespace): + functions = _module_functions() + body = [functions[name] for name in names] + code = compile(ast.Module(body=body, type_ignores=[]), "background_tasks.py", "exec") + exec(code, namespace) + return [namespace[name] for name in names] + + +class LoopHarness: + """Fakes for one run of the loop: a scripted clock, the worker entry points and the wait.""" + + def __init__(self, monkeypatch, *, wake_after=(), max_waits=3, factory_errors=(), hint_errors=(), + sweep_seconds=2.0): + self.now = 100.0 + self.wake_after = list(wake_after) + self.max_waits = max_waits + self.factory_errors = list(factory_errors) + self.hint_errors = list(hint_errors) + self.sweep_seconds = sweep_seconds + self.services = object() + self.events = [] + self.waits = [] + self.sleeps = [] + self.logs = [] + self.factory_calls = [] + self.acquire_lock = lambda name, seconds: {"id": name} + self.release_lock = lambda lock: None + monkeypatch.setattr(delivery_worker, "default_workflow_chat_delivery_services", self.factory) + monkeypatch.setattr(delivery_worker, "process_workflow_chat_delivery_hints", self.hints) + monkeypatch.setattr(delivery_worker, "run_workflow_chat_delivery_sweep", self.sweep) + monkeypatch.setattr(delivery_contract, "wait_for_workflow_chat_delivery_hint", self.wait) + + def factory(self, **kwargs): + self.factory_calls.append(kwargs) + if self.factory_errors: + raise self.factory_errors.pop(0) + return self.services + + def hints(self, services): + self.events.append(("hints", services)) + if self.hint_errors: + raise self.hint_errors.pop(0) + return [] + + def sweep(self, services): + self.events.append(("sweep", services)) + self.now += self.sweep_seconds + return {"locked": True, "processed": 0, "outcomes": {}} + + def _stop_if_done(self): + if len(self.waits) + len(self.sleeps) >= self.max_waits: + raise LoopStopped() + + def wait(self, timeout): + self.waits.append(timeout) + self._stop_if_done() + # A scripted hint wakes the loop early; otherwise the wait runs to its timeout. + self.now += self.wake_after.pop(0) if self.wake_after else timeout + return True + + def sleep(self, seconds): + self.sleeps.append(seconds) + self._stop_if_done() + self.now += seconds + + def namespace(self): + return { + "time": types.SimpleNamespace(monotonic=lambda: self.now, sleep=self.sleep), + "nullcontext": nullcontext, + "logging": logging, + "log_event": lambda message, **kwargs: self.logs.append((message, kwargs)), + "acquire_distributed_task_lock": self.acquire_lock, + "release_distributed_task_lock": self.release_lock, + } + + def run(self, app=None): + (loop,) = load_background_functions([LOOP_NAME], self.namespace()) + with pytest.raises(LoopStopped): + loop(app=app) + + +def test_version_is_at_least_the_delivery_release(): + assert_app_version_at_least("0.261.218") + + +def test_the_loop_binds_services_once_and_throttles_the_sweep(monkeypatch): + harness = LoopHarness(monkeypatch, wake_after=[5], max_waits=3) + + harness.run() + + assert harness.factory_calls == [{ + "acquire_lock": harness.acquire_lock, + "release_lock": harness.release_lock, + }], "the services must be bound once, with the distributed lock helpers" + services = harness.services + assert harness.events == [ + ("hints", services), ("sweep", services), + ("hints", services), + ("hints", services), ("sweep", services), + ], "every wake processes hints, but the sweep runs only once per interval" + # t=100 sweeps (2 s) and waits for the rest of the interval; a hint wakes it at t=107; t=130 + # is the next sweep. + assert harness.waits == [INTERVAL - 2.0, INTERVAL - 7.0, INTERVAL - 2.0] + assert harness.sleeps == [] + assert harness.logs == [] + + +def test_the_wait_never_drops_below_one_second(monkeypatch): + harness = LoopHarness(monkeypatch, max_waits=2, sweep_seconds=INTERVAL + 5.0) + + harness.run() + + assert harness.waits[0] == 1.0, "a sweep that overruns the interval must not leave a zero-second wait" + + +def test_a_failure_sleeps_instead_of_spinning_and_logs_only_the_error_type(monkeypatch): + harness = LoopHarness( + monkeypatch, + max_waits=3, + factory_errors=[RuntimeError(CANARY)], + hint_errors=[ValueError(CANARY)], + ) + + harness.run() + + services = harness.services + assert len(harness.factory_calls) == 2, "a failed bind is retried, and a bound service is kept" + assert harness.events == [("hints", services), ("hints", services), ("sweep", services)] + assert harness.sleeps == [INTERVAL, INTERVAL], "a failure waits a full interval without the hint event" + assert harness.logs == [ + (LOG_MESSAGE, {"level": logging.ERROR, "extra": {"error_type": "RuntimeError"}}), + (LOG_MESSAGE, {"level": logging.ERROR, "extra": {"error_type": "ValueError"}}), + ] + assert CANARY not in repr(harness.logs), "exception text must never reach the log" + + +def test_each_slice_runs_inside_the_app_context(monkeypatch): + harness = LoopHarness(monkeypatch, max_waits=1) + + class FakeApp: + @contextmanager + def app_context(self): + harness.events.append(("enter", None)) + yield + harness.events.append(("exit", None)) + + harness.run(app=FakeApp()) + + services = harness.services + assert harness.events == [("enter", None), ("hints", services), ("sweep", services), ("exit", None)] + + +def test_start_background_task_threads_registers_the_delivery_loop_with_the_app(): + functions = _module_functions() + starter = functions["start_background_task_threads"] + loop_names = sorted({ + node.id for node in ast.walk(starter) + if isinstance(node, ast.Name) and node.id.startswith("run_") + }) + assert LOOP_NAME in loop_names + loop_calls = [] + printed = [] + started = [] + + def recorder(name): + return lambda **kwargs: loop_calls.append((name, kwargs)) + + class FakeThread: + def __init__(self, target=None, daemon=None): + self.target = target + self.daemon = daemon + + def start(self): + started.append(self) + + namespace = {name: recorder(name) for name in loop_names} + namespace.update({"threading": types.SimpleNamespace(Thread=FakeThread), "print": printed.append}) + (start_threads,) = load_background_functions(["start_background_task_threads"], namespace) + app = object() + + threads = start_threads(app=app) + for thread in threads: + thread.target() + + assert all(thread.daemon for thread in started) + assert (LOOP_NAME, {"app": app}) in loop_calls, "the delivery loop must receive the Flask app" + assert loop_calls.count((LOOP_NAME, {"app": app})) == 1 + assert "Workflow chat delivery background task started." in printed + + +def test_the_workflow_scheduler_loop_does_no_delivery_work(): + scheduler = _module_functions()["run_workflow_scheduler_loop"] + names = { + getattr(node, "id", None) or getattr(node, "attr", None) + for node in ast.walk(scheduler) + if isinstance(node, (ast.Name, ast.Attribute)) + } + assert not any("chat_delivery" in str(name) for name in names), ( + "delivery must never run on the workflow scheduler's thread" + ) + + +def real_lock_helpers(): + settings = FakeContainer("settings") + logs = [] + namespace = { + "cosmos_settings_container": settings, + "MatchConditions": MatchConditions, + "datetime": datetime, + "timedelta": timedelta, + "timezone": timezone, + "uuid": uuid, + "socket": socket, + "os": os, + "threading": threading, + "logging": logging, + "log_event": lambda *args, **kwargs: logs.append((args, kwargs)), + } + acquire, release = load_background_functions( + ["_get_lock_holder_id", "_is_expired_timestamp", "acquire_distributed_task_lock", + "release_distributed_task_lock"], + namespace, + )[2:] + return settings, acquire, release, logs + + +def test_the_real_lock_helpers_gate_the_real_sweep(): + settings, acquire, release, logs = real_lock_helpers() + world = make_world() + world.services = replace(world.services, acquire_lock=acquire, release_lock=release) + world.clock.advance(minutes=3) + lock_id = f"background_task_lock_{delivery_contract.SWEEP_LOCK_NAME}" + held_until = (datetime.now(timezone.utc) + timedelta(minutes=5)).isoformat() + settings.put({"id": lock_id, "holder_id": "another-worker", "expires_at": held_until}) + + blocked = delivery_worker.run_workflow_chat_delivery_sweep(world.services) + + assert blocked == {"locked": False, "processed": 0, "outcomes": {}} + assert world.runs.query_calls == [], "another worker's live lease must keep this sweep from querying" + assert settings.get(lock_id)["holder_id"] == "another-worker" + + settings.items.clear() + summary = delivery_worker.run_workflow_chat_delivery_sweep(world.services) + + assert summary["locked"] is True + assert summary["processed"] == 1 + assert summary["outcomes"] == {delivery_worker.OUTCOME_DELIVERED: 1} + assert len(world.delivery_messages()) == 1 + assert settings.get(lock_id) is None, "the sweep must release the lease it took" + assert ("create_item", lock_id) in settings.calls + assert logs == [] + + +def test_the_worker_calls_the_lock_helpers_with_compatible_arguments(): + _settings, acquire, release, _logs = real_lock_helpers() + calls = [] + + def record_acquire(*args, **kwargs): + calls.append(("acquire", args, kwargs)) + return acquire(*args, **kwargs) + + def record_release(*args, **kwargs): + calls.append(("release", args, kwargs)) + return release(*args, **kwargs) + + world = make_world() + world.services = replace(world.services, acquire_lock=record_acquire, release_lock=record_release) + + summary = delivery_worker.run_workflow_chat_delivery_sweep(world.services) + + assert summary["locked"] is True + assert calls[0] == ( + "acquire", (delivery_contract.SWEEP_LOCK_NAME, delivery_contract.SWEEP_LOCK_SECONDS), {}, + ) + assert calls[1][0] == "release" and calls[1][1][0]["task_name"] == delivery_contract.SWEEP_LOCK_NAME + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From 5d06fce9ea3e05d78622efdc54dee0fadd6bc75d Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 01:44:09 -0400 Subject: [PATCH 08/36] Refuse Retry and Edit on messages a workflow run posted (6b-1) Classic Retry and Edit return a closed 400 code for delivered results and notes before any content check or write; the V2 orchestration routes already answer 404 for a delivery message ID. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_workflow_chat_delivery.py | 19 ++ .../single_app/route_backend_conversations.py | 16 ++ .../test_workflow_chat_delivery_refusals.py | 251 ++++++++++++++++++ 3 files changed, 286 insertions(+) create mode 100644 functional_tests/test_workflow_chat_delivery_refusals.py diff --git a/application/single_app/functions_workflow_chat_delivery.py b/application/single_app/functions_workflow_chat_delivery.py index 265180576..16fc67372 100644 --- a/application/single_app/functions_workflow_chat_delivery.py +++ b/application/single_app/functions_workflow_chat_delivery.py @@ -162,6 +162,15 @@ ) UNDELIVERABLE_NOTICE_MESSAGE = "The chat that started this run can't show it anymore. Open the run in Workflows to see the details." EXPIRED_NOTICE_MESSAGE = "The run didn't finish in time to post to the chat. Open it in Workflows to see where it stands." +DELIVERY_RETRY_UNSUPPORTED = 'workflow_delivery_retry_unsupported' +DELIVERY_EDIT_UNSUPPORTED = 'workflow_delivery_edit_unsupported' +_DELIVERY_REFUSALS = { + DELIVERY_RETRY_UNSUPPORTED: ( + "A workflow run posted this message, so it can't be retried here. " + "To run the workflow again, open the run in Workflows." + ), + DELIVERY_EDIT_UNSUPPORTED: "A workflow run posted this message, so it can't be edited. Ask a new question instead.", +} _DEFAULT_WORKFLOW_NAME = 'Workflow' _TEXT_NOISE = re.compile('[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff]') _TIME_ZONE_RE = re.compile(r'[A-Za-z0-9_+\-/]{1,64}') @@ -260,6 +269,13 @@ def is_workflow_delivery_message(message): return isinstance(metadata, Mapping) and isinstance(metadata.get(DELIVERY_METADATA_KEY), Mapping) +def workflow_delivery_refusal_payload(code): + """Return the fixed 400 body for Retry or Edit on a message a workflow run posted.""" + if code not in _DELIVERY_REFUSALS: + code = DELIVERY_RETRY_UNSUPPORTED + return {'error': _DELIVERY_REFUSALS[code], 'code': code}, 400 + + def normalize_requester_roles(roles): if not isinstance(roles, (list, tuple)): return [] @@ -925,6 +941,8 @@ def delivery_log(message, *, level=logging.INFO, status_route=False, run_id=None 'WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY', 'UNDELIVERABLE_NOTICE_MESSAGE', 'EXPIRED_NOTICE_MESSAGE', + 'DELIVERY_RETRY_UNSUPPORTED', + 'DELIVERY_EDIT_UNSUPPORTED', 'clean_catalog_text', 'quoted_workflow_name', 'display_workflow_name', @@ -934,6 +952,7 @@ def delivery_log(message, *, level=logging.INFO, status_route=False, run_id=None 'workflow_delivery_message_id', 'delivery_thread_id', 'is_workflow_delivery_message', + 'workflow_delivery_refusal_payload', 'normalize_requester_roles', 'normalize_model_selection', 'normalize_time_zone', diff --git a/application/single_app/route_backend_conversations.py b/application/single_app/route_backend_conversations.py index 2c8642707..36e30a33d 100644 --- a/application/single_app/route_backend_conversations.py +++ b/application/single_app/route_backend_conversations.py @@ -96,6 +96,12 @@ is_saved_analysis_unavailable, sanitize_saved_analysis_messages, ) +from functions_workflow_chat_delivery import ( + DELIVERY_EDIT_UNSUPPORTED, + DELIVERY_RETRY_UNSUPPORTED, + is_workflow_delivery_message, + workflow_delivery_refusal_payload, +) from functions_workflow_result_masking import message_asks_about_workflow_result, message_uses_workflow_result from functions_workflow_result_reader import WorkflowResultUnavailable, workflow_result_error_payload from utils_cache import invalidate_personal_search_cache @@ -3141,6 +3147,11 @@ def retry_message(message_id): return jsonify({'error': 'Conversation not found'}), 404 elif message_user_id != user_id: return jsonify({'error': 'You can only retry your own messages'}), 403 + + if is_workflow_delivery_message(original_msg): + # A workflow run posted this message, so there is no question to replay. + payload, status = workflow_delivery_refusal_payload(DELIVERY_RETRY_UNSUPPORTED) + return jsonify(payload), status # Get thread info from original message thread_id = original_msg.get('metadata', {}).get('thread_info', {}).get('thread_id') @@ -3364,6 +3375,11 @@ def edit_message(message_id): original_msg = message_results[0] conversation_id = original_msg.get('conversation_id') original_role = original_msg.get('role') + + if is_workflow_delivery_message(original_msg): + # A workflow run posted this message, so there is nothing of the user's to edit. + payload, status = workflow_delivery_refusal_payload(DELIVERY_EDIT_UNSUPPORTED) + return jsonify(payload), status # Only allow editing user messages if original_role != 'user': diff --git a/functional_tests/test_workflow_chat_delivery_refusals.py b/functional_tests/test_workflow_chat_delivery_refusals.py new file mode 100644 index 000000000..6950712be --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_refusals.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_refusals.py +""" +Functional test for refusing Retry and Edit on messages a workflow run posted to chat. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures, in a fresh offline process that boots the real application, that the +classic Retry and Edit routes refuse every message the workflow chat delivery posted (a +delivered result, a failed note, a cancelled note and the note sent when workflow results are +off) with one closed 400 code, before any content check or write, whether the message is +recognised by its ID prefix or by its delivery metadata. A delivered result also carries 6a's +lineage keys, so the test proves the delivery code wins over 6a's Follow up refusal. Another +user's Retry is still refused by ownership first, ordinary turns in the same chat still replay, +and the V2 orchestration retry and edit routes, which take a run ID, answer 404 for a delivery +message ID. Only storage, the model and the network are faked. +""" + +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +APP = ROOT / "application" / "single_app" +TESTS = ROOT / "functional_tests" +sys.path.insert(0, str(APP)) +sys.path.insert(0, str(TESTS)) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + + +SCENARIOS_FINISHED = "WORKFLOW_CHAT_DELIVERY_REFUSAL_SCENARIOS_FINISHED" + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.218") + + +def run_offline_scenarios(): + from copy import deepcopy + from datetime import datetime, timedelta, timezone + from unittest.mock import patch + + from test_support.workflow_result_offline_app import offline_workflow_result_app + + with offline_workflow_result_app({"enable_chat_orchestration": True}) as world: + # Application modules are imported only once the harness has isolated external I/O. + import functions_workflow_chat_delivery as delivery + import functions_workflow_result_reader as reader + import route_backend_conversations as conversations + from test_support.workflow_result_chat import OTHER_USER, RUN_ID, USER, WORKFLOW_ID + + config = world.config + messages, chats = config.cosmos_messages_container, config.cosmos_conversations_container + owner, stranger = world.signed_in(USER, "Owner"), world.signed_in(OTHER_USER, "Stranger") + + retry_refusal, retry_status = delivery.workflow_delivery_refusal_payload(delivery.DELIVERY_RETRY_UNSUPPORTED) + edit_refusal, edit_status = delivery.workflow_delivery_refusal_payload(delivery.DELIVERY_EDIT_UNSUPPORTED) + follow_up_refusal, _ = reader.workflow_result_error_payload( + reader.WorkflowResultUnavailable("workflow_result_retry_unsupported") + ) + require( + retry_status == edit_status == 400 + and retry_refusal["code"] == "workflow_delivery_retry_unsupported" + and edit_refusal["code"] == "workflow_delivery_edit_unsupported" + and retry_refusal != follow_up_refusal, + f"Unexpected refusal bodies: {retry_refusal} {edit_refusal}", + ) + unknown = delivery.workflow_delivery_refusal_payload("anything_else") + require(unknown == (retry_refusal, 400), f"An unknown code wasn't closed: {unknown}") + + checks, records = [], [] + real_check = conversations.check_chat_content + + def check(content, kind, **kwargs): + checks.append(content) + return real_check(content, kind, **kwargs) + + def record(*args, **kwargs): + records.append(args) + + world.stack.enter_context(patch.object(conversations, "check_chat_content", check)) + world.stack.enter_context(patch.object(conversations, "record_blocked_chat_attempt", record)) + + conversation_id = "conv-delivery-refusals" + asked_at = datetime.now(timezone.utc) - timedelta(hours=3) + chats.upsert_item({ + "id": conversation_id, "user_id": USER, "title": "Digest", "last_updated": asked_at.isoformat(), + "context": [], "tags": [], "classification": [], + }) + + def seeded(message_id, role, content, at, thread_id, previous_thread_id=None, **metadata): + row = { + "id": message_id, "conversation_id": conversation_id, "role": role, "content": content, + "timestamp": at.isoformat(), "model_deployment_name": "gpt-4o" if role == "assistant" else None, + "metadata": { + "user_info": {"user_id": USER, "display_name": "Owner"}, + "thread_info": { + "thread_id": thread_id, "previous_thread_id": previous_thread_id, + "active_thread": True, "thread_attempt": 1, + }, + **metadata, + }, + } + messages.upsert_item(row) + return deepcopy(row) + + # The turn that started the run: an ordinary question and the plan's answer. + question = seeded("msg-ask-q", "user", "Run my weekly digest.", asked_at, "thread-ask") + answer = seeded( + "msg-ask-a", "assistant", "I started the run.", asked_at + timedelta(seconds=5), "thread-ask", + ) + + def posted(kind, generation, run_status, *, message_id=None, with_metadata=True, **extra): + message_id = message_id or delivery.workflow_delivery_message_id(RUN_ID, conversation_id, generation) + metadata = dict(extra) + if with_metadata: + metadata[delivery.DELIVERY_METADATA_KEY] = delivery.build_delivery_metadata( + kind=kind, workflow_id=WORKFLOW_ID, run_id=RUN_ID, generation=generation, + run_status=run_status, orchestration_run_id="orch-run-1", step_id="step-1", + requested_at=asked_at.isoformat(), + ) + return seeded( + message_id, "assistant", f"A {kind} message.", asked_at + timedelta(hours=1, minutes=generation), + delivery.delivery_thread_id(message_id), "thread-ask", token_usage={}, **metadata, + ) + + delivered = { + "result": posted( + delivery.KIND_RESULT, 1, "completed", + workflow_result={"version": 1, "available": True}, workflow_result_contexts=[world.context], + ), + "failed note": posted(delivery.KIND_FAILED, 2, "failed"), + "cancelled note": posted(delivery.KIND_CANCELLED, 3, "cancelled"), + "results-off note": posted(delivery.KIND_STATUS, 4, "completed"), + "metadata only": posted(delivery.KIND_FAILED, 5, "failed", message_id="msg-delivery-by-metadata"), + "prefix only": posted(delivery.KIND_STATUS, 6, "completed", with_metadata=False), + } + require( + all(delivery.is_workflow_delivery_message(row) for row in delivered.values()) + and not delivery.is_workflow_delivery_message(question) + and not delivery.is_workflow_delivery_message(answer), + "The seeded messages aren't classified as expected.", + ) + require( + conversations.message_asks_about_workflow_result(delivered["result"]), + "The delivered result doesn't carry 6a's lineage, so the precedence check proves nothing.", + ) + + def stored(): + return deepcopy(messages.items), deepcopy(chats.items) + + def replay(client, kind, message, **body): + return client.post(f"/api/message/{message['id']}/{kind}", json=body) + + for label, message in delivered.items(): + for kind, body, refusal in ( + ("retry", {}, retry_refusal), + ("edit", {"content": "Say it differently."}, edit_refusal), + ): + before = stored() + checks.clear() + records.clear() + refused = replay(owner, kind, message, **body) + outcome = (refused.status_code, refused.get_json(silent=True)) + require( + outcome == (400, refusal), + f"{kind} of the {label}: {refused.status_code} {refused.get_data(as_text=True)[:300]}", + ) + require(not checks and not records, f"{kind} of the {label} was checked: {checks} {records}") + require(stored() == before, f"{kind} of the {label} wrote to storage.") + + # Another user's Retry is refused by ownership before the message is inspected. Edit refuses + # before ownership, as its role check already does for any assistant message, so a stranger + # learns nothing a message ID with this prefix doesn't already say. + before = stored() + foreign_retry = replay(stranger, "retry", delivered["result"]) + require( + foreign_retry.status_code == 403, + f"A stranger's retry: {foreign_retry.status_code} {foreign_retry.get_data(as_text=True)[:300]}", + ) + foreign_edit = replay(stranger, "edit", delivered["failed note"], content="Mine now.") + require( + (foreign_edit.status_code, foreign_edit.get_json(silent=True)) == (400, edit_refusal), + f"A stranger's edit: {foreign_edit.status_code} {foreign_edit.get_data(as_text=True)[:300]}", + ) + require(stored() == before and not checks, "A stranger's request wrote to storage or was checked.") + + # The V2 orchestration retry and edit take an orchestration run ID, so a delivery message ID + # names no run. + delivery_id = delivered["result"]["id"] + v2_retry = owner.post(f"/api/v2/orchestration/runs/{delivery_id}/retry", json={ + "conversation_id": conversation_id, "submission_id": "submission-1", "expected_version": "version-1", + }) + require( + v2_retry.status_code == 404, + f"V2 retry with a delivery ID: {v2_retry.status_code} {v2_retry.get_data(as_text=True)[:300]}", + ) + v2_edit = owner.post(f"/api/v2/orchestration/runs/{delivery_id}/edit", json={ + "conversation_id": conversation_id, + }) + require( + v2_edit.status_code == 404, + f"V2 edit with a delivery ID: {v2_edit.status_code} {v2_edit.get_data(as_text=True)[:300]}", + ) + require(stored() == before, "A V2 retry or edit with a delivery ID wrote to storage.") + + # Ordinary turns in the same chat still replay, and each replays its own question. + checks.clear() + retried = replay(owner, "retry", answer) + require( + retried.status_code == 200 and retried.get_json().get("success") is True, + f"An ordinary retry failed: {retried.status_code} {retried.get_data(as_text=True)[:300]}", + ) + edited = replay(owner, "edit", question, content="Run my monthly digest.") + require( + edited.status_code == 200 and edited.get_json().get("success") is True, + f"An ordinary edit failed: {edited.status_code} {edited.get_data(as_text=True)[:300]}", + ) + require(checks == [question["content"], "Run my monthly digest."], f"Unexpected checks: {checks}") + + require(not world.normal_chat_clients, "The normal chat model was used.") + require(not world.network_attempts, f"The routes attempted network access: {world.network_attempts}") + print(SCENARIOS_FINISHED) + + +def test_retry_and_edit_refuse_every_message_a_workflow_run_posted(): + result = subprocess.run( + [sys.executable, *(["-O"] if sys.flags.optimize else []), str(Path(__file__).resolve()), "--offline"], + # app.py switches the child's output to UTF-8, which the parent's locale codec can't always decode. + cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=240, + ) + output = result.stdout[-3000:] + result.stderr[-6000:] + + require(result.returncode == 0, output) + require(SCENARIOS_FINISHED in result.stdout.splitlines(), output) + + +if __name__ == "__main__": + if "--offline" in sys.argv: + run_offline_scenarios() + else: + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From d5abb050c59163b8534f18346dcba1ff66694d98 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 03:11:30 -0400 Subject: [PATCH 09/36] Add the workflow chat delivery status route (6b-1) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_workflow_chat_delivery.py | 28 +- ...functions_workflow_chat_delivery_status.py | 698 +++++++++++++++ .../single_app/route_backend_orchestration.py | 46 +- .../test_route_blueprint_policy_inventory.py | 27 +- ...t_route_unauthenticated_policy_contract.py | 23 +- ...est_workflow_chat_delivery_status_route.py | 822 ++++++++++++++++++ 6 files changed, 1629 insertions(+), 15 deletions(-) create mode 100644 application/single_app/functions_workflow_chat_delivery_status.py create mode 100644 functional_tests/test_workflow_chat_delivery_status_route.py diff --git a/application/single_app/functions_workflow_chat_delivery.py b/application/single_app/functions_workflow_chat_delivery.py index 16fc67372..f19ad41d3 100644 --- a/application/single_app/functions_workflow_chat_delivery.py +++ b/application/single_app/functions_workflow_chat_delivery.py @@ -679,19 +679,22 @@ def assemble_delivery_content(label, body, disclosure=''): return '\n\n'.join(parts) +_FAILURE_REASONS = { + 'failed': 'the run stopped before it finished', + 'invalid': "the workflow couldn't run as defined", + 'incomplete': "some of its steps didn't finish", + 'skipped': 'no new or changed files were detected', + REASON_DEADLINE_EXCEEDED: 'it reached its time limit', + 'execution_budget_exceeded': 'it reached its execution limit', + 'repeat_iteration_limit': 'it reached its repeat limit', + 'm365_authorization': 'Microsoft 365 access needs to be reconnected', + 'authorization': 'your workflow access must be restored', +} +FAILURE_REASON_CODES = frozenset(_FAILURE_REASONS) + + def failure_reason_text(code): - reasons = { - 'failed': 'the run stopped before it finished', - 'invalid': "the workflow couldn't run as defined", - 'incomplete': "some of its steps didn't finish", - 'skipped': 'no new or changed files were detected', - REASON_DEADLINE_EXCEEDED: 'it reached its time limit', - 'execution_budget_exceeded': 'it reached its execution limit', - 'repeat_iteration_limit': 'it reached its repeat limit', - 'm365_authorization': 'Microsoft 365 access needs to be reconnected', - 'authorization': 'your workflow access must be restored', - } - return reasons.get(code, reasons['failed']) + return _FAILURE_REASONS.get(code, _FAILURE_REASONS['failed']) def delivery_note_text(kind, workflow_name, *, reason_code=None): @@ -968,6 +971,7 @@ def delivery_log(message, *, level=logging.INFO, status_route=False, run_id=None 'delivery_label', 'assemble_delivery_content', 'delivery_note_text', + 'FAILURE_REASON_CODES', 'failure_reason_text', 'notice_workflow_name', 'notice_title', diff --git a/application/single_app/functions_workflow_chat_delivery_status.py b/application/single_app/functions_workflow_chat_delivery_status.py new file mode 100644 index 000000000..2098eb2dc --- /dev/null +++ b/application/single_app/functions_workflow_chat_delivery_status.py @@ -0,0 +1,698 @@ +# functions_workflow_chat_delivery_status.py +"""Owner-only status rows for the workflow runs a user's chats started. + +Version: 0.261.218 +Implemented in: 0.261.218 + +Backs ``GET /api/v2/orchestration/workflow-runs/status``, the batched route the V2 run card polls. +Every read is a single-partition query in the signed-in user's own partition that projects only +the fields a row needs. In-flight rows get a bounded number of live runtime reads; finished rows +use the runtime projection stored on the run. Nothing is written, and nothing in a row comes from +a run's output, its error text, or an exception. +""" + +import logging +import re +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Callable + +from functions_m365_workflow_binding import M365_WAITING_STATES +from functions_workflow_alert_safety import sanitize_workflow_alert_record +from functions_workflow_chat_delivery import ( + CHAT_TRIGGER_SOURCE, + DELIVERY_MESSAGE_ID_PREFIX, + FAILED_RUN_STATES, + FAILURE_REASON_CODES, + NAME_MAX_LENGTH, + OPEN_STATUSES, + REASON_ACCESS_LOST, + REASON_CHAT_UNAVAILABLE, + REASON_CONTENT_BLOCKED, + REASON_DEADLINE_EXCEEDED, + REASON_DELIVERY_FAILED, + REASON_EXPIRED_BEFORE_DELIVERY, + REASON_RESULT_UNAVAILABLE, + REASON_RESULTS_OFF, + REASON_RUNTIME_MISSING, + REASON_WORKFLOW_DELETED, + RESULT_RUN_STATES, + RUN_DOCUMENT_TERMINAL_STATUSES, + RUNTIME_ACTIVE_STATES, + RUNTIME_RESUMABLE_STATES, + RUNTIME_TERMINAL_STATES, + RUNTIME_WAITING_STATES, + STATUS_DELIVERED, + STATUS_EXPIRED, + STATUS_READY, + STATUSES, + WORKFLOW_SCOPE, + clean_catalog_text, + delivery_log, + failure_reason_text, + parse_delivery_timestamp, + utc_now, +) +from functions_workflow_definitions import workflow_definition_for_editor, workflow_definition_revision + + +CONVERSATION_LIMIT = 20 +GLOBAL_LIMIT = 50 +LIVE_READ_LIMIT = 10 +WORKFLOW_READ_LIMIT = 20 +RECENT_DELIVERY_SECONDS = 600 + +INVALID_CONVERSATION_CODE = 'invalid_conversation_id' +STATUS_UNAVAILABLE_CODE = 'workflow_run_status_unavailable' +_ERRORS = { + INVALID_CONVERSATION_CODE: ('The conversation ID is not valid.', 400), + STATUS_UNAVAILABLE_CODE: ("Workflow run status isn't available right now. Try again later.", 503), +} + +ROW_STATUSES = ('queued', 'running', 'waiting', 'completed', 'completed_partial', 'failed', 'cancelled', 'expired') +ROW_PHASES = ('running', 'needs_you', 'finished', 'failed', 'cancelled') +DELIVERY_NOT_APPLICABLE = 'not_applicable' +DELIVERY_ROW_STATUSES = ('pending', 'delivering', 'delivered', 'undeliverable', 'expired', DELIVERY_NOT_APPLICABLE) +DELIVERY_REASONS = frozenset({ + REASON_CHAT_UNAVAILABLE, + REASON_ACCESS_LOST, + REASON_WORKFLOW_DELETED, + REASON_RUNTIME_MISSING, + REASON_DELIVERY_FAILED, + REASON_EXPIRED_BEFORE_DELIVERY, + REASON_CONTENT_BLOCKED, + REASON_RESULTS_OFF, + REASON_RESULT_UNAVAILABLE, + REASON_DEADLINE_EXCEEDED, +}) + +RETRY_BLOCKED_WORKFLOW_DELETED = 'workflow_deleted' +RETRY_BLOCKED_ALREADY_RUNNING = 'workflow_already_running' +RETRY_BLOCKED_DEFINITION_CHANGED = 'workflow_definition_changed' +RETRY_BLOCKED_NOT_RESUMABLE = 'not_resumable' +RETRY_BLOCKED_DEADLINE_EXCEEDED = 'deadline_exceeded' +RETRY_BLOCKED_UNAVAILABLE = 'retry_unavailable' +RETRY_BLOCKED_CODES = frozenset({ + RETRY_BLOCKED_WORKFLOW_DELETED, + RETRY_BLOCKED_ALREADY_RUNNING, + RETRY_BLOCKED_DEFINITION_CHANGED, + RETRY_BLOCKED_NOT_RESUMABLE, + RETRY_BLOCKED_DEADLINE_EXCEEDED, + RETRY_BLOCKED_UNAVAILABLE, +}) + +WAITING_ACTION_APPROVE = 'approve' +WAITING_ACTION_RECONNECT = 'reconnect' +WAITING_ACTION_OPEN_RUN = 'open_run' + +_KNOWN_STATES = RUNTIME_ACTIVE_STATES | RUNTIME_WAITING_STATES | RUNTIME_TERMINAL_STATES | frozenset({'canceled'}) +_CANCELLED_STATES = frozenset({'cancelled', 'canceled'}) +_DEFAULT_WORKFLOW_NAME = 'Workflow' +_CONVERSATION_ID = re.compile(r'[A-Za-z0-9_-]{1,128}') +_CODE_MAX_LENGTH = 64 +_ID_MAX_LENGTH = 256 +_SECONDS_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + +_READ_OK = 'ok' +_READ_GONE = 'gone' +_READ_DELETING = 'deleting' +_READ_ERROR = 'error' +_READ_OVER_CAP = 'over_cap' + +_PROJECTION = ', '.join(( + 'c.id AS run_id', + 'c.workflow_id AS workflow_id', + 'c.workflow_name AS workflow_name', + 'c.status AS status', + 'c.started_at AS started_at', + 'c.completed_at AS completed_at', + 'c.progress AS progress', + 'c.runtime_version AS runtime_version', + 'c.definition_revision AS definition_revision', + 'c.chat_invocation.conversation_id AS conversation_id', + 'c.chat_invocation.orchestration_run_id AS orchestration_run_id', + 'c.chat_invocation.step_id AS step_id', + 'c.chat_invocation.requested_at AS requested_at', + 'c.runtime.version AS projection_version', + 'c.runtime.state AS projection_state', + 'c.runtime.phase AS projection_phase', + 'c.runtime.progress AS projection_progress', + 'c.runtime.can_resume AS projection_can_resume', + 'c.runtime.deleted AS projection_deleted', + 'c.runtime.limits.deadline_at AS projection_deadline_at', + 'c.runtime.gate.id AS projection_gate_id', + 'c.runtime.gate.reason_code AS projection_gate_reason_code', + 'c.chat_delivery.status AS delivery_status', + 'c.chat_delivery.generation AS delivery_generation', + 'c.chat_delivery.message_id AS delivery_message_id', + 'c.chat_delivery.delivered_at AS delivery_delivered_at', + 'c.chat_delivery.outcome_reason AS delivery_outcome_reason', +)) +_OWNER_FILTER = 'c.user_id = @user_id AND c.trigger_source = @source AND IS_DEFINED(c.chat_invocation)' +_NEWEST_FIRST = 'ORDER BY c.chat_invocation.requested_at DESC' +CONVERSATION_QUERY = ( + f'SELECT TOP {CONVERSATION_LIMIT + 1} {_PROJECTION} FROM c ' + f'WHERE {_OWNER_FILTER} AND c.chat_invocation.conversation_id = @conversation_id ' + f'{_NEWEST_FIRST}' +) +GLOBAL_QUERY = ( + f'SELECT TOP {GLOBAL_LIMIT + 1} {_PROJECTION} FROM c ' + f'WHERE {_OWNER_FILTER} ' + 'AND (NOT ARRAY_CONTAINS(@terminal, c.status) ' + 'OR ARRAY_CONTAINS(@open, c.chat_delivery.status) ' + 'OR (IS_STRING(c.chat_delivery.delivered_at) AND c.chat_delivery.delivered_at >= @recent)) ' + f'{_NEWEST_FIRST}' +) + + +class WorkflowRunStatusError(Exception): + """A closed status-route failure: a code, its HTTP status and a fixed sentence.""" + + def __init__(self, code): + if code not in _ERRORS: + code = STATUS_UNAVAILABLE_CODE + super().__init__(code) + self.code = code + self.status = _ERRORS[code][1] + + def payload(self): + return {'error': _ERRORS[self.code][0], 'code': self.code}, self.status + + +@dataclass(frozen=True) +class WorkflowRunStatusServices: + """The reads one status request needs. Tests bind fakes; the route binds the real ones.""" + + runs: Any + workflows: Any + get_settings: Callable[[], Any] + settings_gate: Callable[[Any], Any] + live_status: Callable[..., Any] + clock: Callable[[], datetime] = utc_now + + +def default_workflow_run_status_services(): + """Bind the personal workflow containers and runtime reader on first use. + + Imported here so importing this module never loads config or the runtime. + """ + import config + from functions_orchestration_workflow_context import workflow_run_settings_gate + from functions_settings import get_settings + from functions_workflow_runtime import workflow_runtime_status + + return WorkflowRunStatusServices( + runs=config.cosmos_personal_workflow_runs_container, + workflows=config.cosmos_personal_workflows_container, + get_settings=get_settings, + settings_gate=workflow_run_settings_gate, + live_status=workflow_runtime_status, + clock=utc_now, + ) + + +def _identifier(value, max_length=_ID_MAX_LENGTH): + """A stored id as it is, or None: the same shape the runtime store accepts for its ids.""" + if ( + not isinstance(value, str) + or not 0 < len(value) <= max_length + or value != value.strip() + or any(ord(char) < 32 or 0xD800 <= ord(char) <= 0xDFFF for char in value) + ): + return None + return value + + +def _code(value): + if not isinstance(value, str): + return None + text = value.strip().lower() + if not text or len(text) > _CODE_MAX_LENGTH: + return None + return text + + +def _known_state(value): + state = _code(value) + return state if state in _KNOWN_STATES else None + + +def _count(value): + return value if type(value) is int and value >= 0 else None + + +def _seconds(value): + moment = parse_delivery_timestamp(value) + return moment.strftime(_SECONDS_FORMAT) if moment is not None else None + + +def parse_status_conversation_id(values): + """The one optional ``conversation_id`` query value, trimmed; a repeated or malformed one is refused.""" + if values is None: + return None + if isinstance(values, str): + values = [values] + values = list(values) + if not values: + return None + if len(values) > 1: + raise WorkflowRunStatusError(INVALID_CONVERSATION_CODE) + text = values[0].strip() if isinstance(values[0], str) else '' + if not _CONVERSATION_ID.fullmatch(text): + raise WorkflowRunStatusError(INVALID_CONVERSATION_CODE) + return text + + +def status_query(user_id, conversation_id, now): + """The query text, its parameters and the row limit. One partition, newest request first.""" + parameters = [ + {'name': '@user_id', 'value': user_id}, + {'name': '@source', 'value': CHAT_TRIGGER_SOURCE}, + ] + if conversation_id is not None: + parameters.append({'name': '@conversation_id', 'value': conversation_id}) + return CONVERSATION_QUERY, parameters, CONVERSATION_LIMIT + recent = (now - timedelta(seconds=RECENT_DELIVERY_SECONDS)).strftime('%Y-%m-%dT%H:%M:%S.%fZ') + parameters.extend(( + {'name': '@terminal', 'value': sorted(RUN_DOCUMENT_TERMINAL_STATUSES)}, + {'name': '@open', 'value': sorted(OPEN_STATUSES)}, + {'name': '@recent', 'value': recent}, + )) + return GLOBAL_QUERY, parameters, GLOBAL_LIMIT + + +class _WorkflowRead: + __slots__ = ('kind', 'workflow') + + def __init__(self, kind, workflow=None): + self.kind = kind + self.workflow = workflow + + +class _RequestReads: + """One request's read budget. + + One workflow read per distinct workflow (at most ``WORKFLOW_READ_LIMIT``), counted apart from + the live runtime reads (at most ``LIVE_READ_LIMIT``). + """ + + def __init__(self, services, user_id): + self.services = services + self.user_id = user_id + self.workflows = {} + self.live_reads = 0 + + def workflow(self, workflow_id): + if workflow_id in self.workflows: + return self.workflows[workflow_id] + if len(self.workflows) >= WORKFLOW_READ_LIMIT: + return _WorkflowRead(_READ_OVER_CAP) + read = self._read_workflow(workflow_id) + self.workflows[workflow_id] = read + return read + + def _read_workflow(self, workflow_id): + """The workflow in the form the runtime compares revisions on, or why it can't be used.""" + try: + raw = self.services.workflows.read_item(item=workflow_id, partition_key=self.user_id) + except Exception as exc: + if getattr(exc, 'status_code', None) == 404: + return _WorkflowRead(_READ_GONE) + delivery_log( + 'Workflow read failed.', level=logging.WARNING, status_route=True, + error_type=type(exc).__name__, + ) + return _WorkflowRead(_READ_ERROR) + if not isinstance(raw, Mapping) or raw.get('user_id') != self.user_id or raw.get('group_id'): + return _WorkflowRead(_READ_GONE) + if raw.get('deleting') or raw.get('status') == 'deleting': + return _WorkflowRead(_READ_DELETING) + try: + # The same editor form get_personal_workflow returns, which decide_workflow_runtime compares. + cleaned = {key: value for key, value in raw.items() if not str(key).startswith('_')} + workflow = workflow_definition_for_editor(sanitize_workflow_alert_record(cleaned)) + except Exception as exc: + delivery_log( + 'Workflow read failed.', level=logging.WARNING, status_route=True, + error_type=type(exc).__name__, + ) + return _WorkflowRead(_READ_ERROR) + return _WorkflowRead(_READ_OK, workflow) + + def live(self, workflow, run_id): + if self.live_reads >= LIVE_READ_LIMIT: + return None + self.live_reads += 1 + try: + projection = self.services.live_status(workflow, run_id, reader_user_id=self.user_id) + except Exception as exc: + delivery_log( + 'Live run status read failed; using the stored status.', level=logging.WARNING, + status_route=True, run_id=run_id, error_type=type(exc).__name__, + ) + return None + return projection if isinstance(projection, Mapping) else None + + +def _view(*, version, state, phase, progress, can_resume, deleted, deadline_at, gate_id, gate_reason_code): + return { + 'version': _count(version), + 'state': _known_state(state), + 'phase': _code(phase), + 'progress': progress if isinstance(progress, Mapping) else None, + 'can_resume': can_resume if type(can_resume) is bool else None, + 'deleted': deleted is True, + 'deadline_at': deadline_at, + 'gate_id': _identifier(gate_id), + 'gate_reason_code': _code(gate_reason_code), + } + + +def _stored_view(item): + return _view( + version=item.get('projection_version'), + state=item.get('projection_state'), + phase=item.get('projection_phase'), + progress=item.get('projection_progress'), + can_resume=item.get('projection_can_resume'), + deleted=item.get('projection_deleted'), + deadline_at=item.get('projection_deadline_at'), + gate_id=item.get('projection_gate_id'), + gate_reason_code=item.get('projection_gate_reason_code'), + ) + + +def _live_view(projection): + limits = projection.get('limits') if isinstance(projection.get('limits'), Mapping) else {} + gate = projection.get('gate') if isinstance(projection.get('gate'), Mapping) else {} + return _view( + version=projection.get('version'), + state=projection.get('state'), + phase=projection.get('phase'), + progress=projection.get('progress'), + can_resume=projection.get('can_resume'), + deleted=projection.get('deleted'), + deadline_at=limits.get('deadline_at'), + gate_id=gate.get('id'), + gate_reason_code=gate.get('reason_code'), + ) + + +def _deadline_paused(state, view): + return state == 'paused' and REASON_DEADLINE_EXCEEDED in (view['gate_reason_code'], view['phase']) + + +def _status_and_phase(state, *, expired): + if expired: + return 'expired', 'failed' + if state in ('queued', 'running'): + return state, 'running' + if state in RESULT_RUN_STATES: + return state, 'finished' + if state in FAILED_RUN_STATES or state == 'skipped': + return 'failed', 'failed' + if state in _CANCELLED_STATES: + return 'cancelled', 'cancelled' + if state in RUNTIME_WAITING_STATES and state != 'waiting_recovery': + return 'waiting', 'needs_you' + # cancelling, waiting_recovery, and any state this module doesn't know yet. + return 'running', 'running' + + +def _waiting(state, view): + if state == 'waiting_approval': + reason, action = 'approval', WAITING_ACTION_APPROVE + elif state == 'awaiting_sign_in': + reason, action = 'microsoft_365_reconnect', WAITING_ACTION_RECONNECT + elif state in M365_WAITING_STATES: + reason, action = 'microsoft_365_approval', WAITING_ACTION_OPEN_RUN + elif state == 'waiting_output': + reason, action = 'output_review', WAITING_ACTION_OPEN_RUN + elif state == 'waiting_recovery': + reason, action = 'recovery', WAITING_ACTION_OPEN_RUN + elif state == 'paused': + reason = REASON_DEADLINE_EXCEEDED if _deadline_paused(state, view) else 'paused' + action = WAITING_ACTION_OPEN_RUN + else: + return None + return {'reason': reason, 'action': action, 'gate_id': view['gate_id']} + + +def _steps(*candidates): + """Steps finished and steps in all, from the first ``{completed, total}`` progress that has both.""" + for progress in candidates: + if not isinstance(progress, Mapping): + continue + completed = _count(progress.get('completed')) + total = _count(progress.get('total')) + if completed is not None and total is not None: + return min(completed, total), total + return None, None + + +def _elapsed_seconds(started_at, end): + started = parse_delivery_timestamp(started_at) + if started is None or end is None: + return None + return max(0, int((end - started).total_seconds())) + + +def _delivery(item): + status = item.get('delivery_status') + if status not in STATUSES: + return { + 'status': DELIVERY_NOT_APPLICABLE, + 'generation': None, + 'message_id': None, + 'delivered_at': None, + 'reason': None, + } + delivered = status == STATUS_DELIVERED + message_id = _identifier(item.get('delivery_message_id')) + reason = item.get('delivery_outcome_reason') + return { + 'status': 'pending' if status == STATUS_READY else status, + 'generation': _count(item.get('delivery_generation')), + 'message_id': message_id if delivered and message_id and message_id.startswith(DELIVERY_MESSAGE_ID_PREFIX) else None, + 'delivered_at': _seconds(item.get('delivery_delivered_at')) if delivered else None, + 'reason': reason if reason in DELIVERY_REASONS else None, + } + + +def _failure(status, state, view): + """The closed failure code and its fixed sentence; never the run's own error text.""" + if status == 'expired': + code = REASON_DEADLINE_EXCEEDED + elif status == 'failed': + candidate = view['gate_reason_code'] or state + code = candidate if candidate in FAILURE_REASON_CODES else 'failed' + else: + return None, None + text = failure_reason_text(code) + return text[:1].upper() + text[1:] + '.', code + + +def _retry_blocked(view, runtime_version, item, workflow_read, run_id, delivery_status, now): + """Why the durable resume route would refuse this run now, or None when it should accept it. + + The checks mirror ``decide_workflow_runtime`` and ``journal_request('resume')`` on the stored + projection, so the answer is a snapshot: the resume route can still refuse. + """ + if view['can_resume'] is None or runtime_version is None: + return RETRY_BLOCKED_UNAVAILABLE + if view['deleted']: + return RETRY_BLOCKED_WORKFLOW_DELETED + if not view['can_resume']: + return RETRY_BLOCKED_NOT_RESUMABLE + deadline = parse_delivery_timestamp(view['deadline_at']) + if (deadline is not None and deadline <= now) or delivery_status == STATUS_EXPIRED: + return RETRY_BLOCKED_DEADLINE_EXCEEDED + if workflow_read is None or workflow_read.kind in (_READ_ERROR, _READ_OVER_CAP): + return RETRY_BLOCKED_UNAVAILABLE + if workflow_read.kind in (_READ_GONE, _READ_DELETING): + return RETRY_BLOCKED_WORKFLOW_DELETED + revision = item.get('definition_revision') + if not isinstance(revision, str) or not revision: + return RETRY_BLOCKED_UNAVAILABLE + try: + current = workflow_definition_revision(workflow_read.workflow) + except Exception: + return RETRY_BLOCKED_UNAVAILABLE + if current != revision: + return RETRY_BLOCKED_DEFINITION_CHANGED + if workflow_read.workflow.get('active_run_id') not in (None, '', run_id): + return RETRY_BLOCKED_ALREADY_RUNNING + return None + + +def _status_row(item, reads, now): + if not isinstance(item, Mapping): + return None + run_id = _identifier(item.get('run_id')) + workflow_id = _identifier(item.get('workflow_id')) + conversation_id = _identifier(item.get('conversation_id')) + if not run_id or not workflow_id or not conversation_id: + return None + + stored = _stored_view(item) + stored_state = stored['state'] or _known_state(item.get('status')) + in_flight = stored_state not in RUN_DOCUMENT_TERMINAL_STATUSES + workflow_read = None + if in_flight or stored_state in RUNTIME_RESUMABLE_STATES: + workflow_read = reads.workflow(workflow_id) + + view = stored + live = False + if in_flight and workflow_read.kind == _READ_OK: + projection = reads.live(workflow_read.workflow, run_id) + if projection is not None: + view = _live_view(projection) + live = True + state = view['state'] or stored_state + terminal = state in RUN_DOCUMENT_TERMINAL_STATUSES + + if live and view['version'] is not None: + runtime_version = view['version'] + elif _count(item.get('runtime_version')) is not None: + runtime_version = item['runtime_version'] + else: + runtime_version = stored['version'] + + delivery = _delivery(item) + raw_delivery_status = item.get('delivery_status') + expired = not terminal and (raw_delivery_status == STATUS_EXPIRED or _deadline_paused(state, view)) + status, phase = _status_and_phase(state, expired=expired) + waiting = _waiting(state, view) + error, error_code = _failure(status, state, view) + + retry_blocked = None + if state in RUNTIME_RESUMABLE_STATES: + retry_blocked = _retry_blocked(view, runtime_version, item, workflow_read, run_id, raw_delivery_status, now) + step_index, step_count = _steps(view['progress'], stored['progress'], item.get('progress')) + completed_at = _seconds(item.get('completed_at')) if terminal else None + + return { + 'workflow_id': workflow_id, + 'workflow_scope': WORKFLOW_SCOPE, + 'run_id': run_id, + 'workflow_name': clean_catalog_text(item.get('workflow_name'), NAME_MAX_LENGTH) or _DEFAULT_WORKFLOW_NAME, + 'conversation_id': conversation_id, + 'orchestration_run_id': _identifier(item.get('orchestration_run_id')), + 'step_id': _identifier(item.get('step_id')), + 'requested_at': _seconds(item.get('requested_at')), + 'status': status, + 'phase': phase, + 'runtime_version': runtime_version, + 'step_index': step_index, + 'step_count': step_count, + 'step_label': None, + 'started_at': _seconds(item.get('started_at')), + 'completed_at': completed_at, + 'elapsed_seconds': _elapsed_seconds( + item.get('started_at'), + parse_delivery_timestamp(completed_at) if terminal else now, + ), + 'waiting': waiting, + 'delivery': delivery, + 'error': error, + 'error_code': error_code, + 'retry_blocked': retry_blocked, + 'actions': { + 'cancel': not terminal and not expired and state != 'cancelling', + 'retry': state in RUNTIME_RESUMABLE_STATES and retry_blocked is None, + 'approve': bool( + waiting and waiting['action'] == WAITING_ACTION_APPROVE and waiting['gate_id'] and not expired + ), + 'open_run': True, + }, + 'live': live, + } + + +def _request_now(services): + try: + moment = parse_delivery_timestamp(services.clock()) + except Exception: + moment = None + return moment or utc_now() + + +def workflow_run_status_payload(user_id, conversation_id_values=None, *, services=None): + """The status route's 200 body for ``user_id``, or a ``WorkflowRunStatusError``. + + ``conversation_id_values`` is every ``conversation_id`` the request sent. With one, the rows + are that chat's runs; with none, they are the user's runs still in flight, still waiting to be + delivered, or delivered in the last ten minutes. Rows come back even when chats can no longer + start workflows, so an in-flight card doesn't vanish when an admin changes the setting. + """ + conversation_id = parse_status_conversation_id(conversation_id_values) + if _identifier(user_id) is None: + raise WorkflowRunStatusError(STATUS_UNAVAILABLE_CODE) + services = services or default_workflow_run_status_services() + now = _request_now(services) + try: + available = services.settings_gate(services.get_settings()) is None + except Exception as exc: + delivery_log( + 'Workflow run status settings read failed.', level=logging.WARNING, status_route=True, + error_type=type(exc).__name__, + ) + raise WorkflowRunStatusError(STATUS_UNAVAILABLE_CODE) from None + + query, parameters, limit = status_query(user_id, conversation_id, now) + try: + items = list(services.runs.query_items(query=query, parameters=parameters, partition_key=user_id)) + except Exception as exc: + delivery_log( + 'Workflow run status query failed.', level=logging.WARNING, status_route=True, + conversation_id=conversation_id, error_type=type(exc).__name__, + ) + raise WorkflowRunStatusError(STATUS_UNAVAILABLE_CODE) from None + + reads = _RequestReads(services, user_id) + rows = [] + for item in items[:limit]: + row = _status_row(item, reads, now) + if row is not None: + rows.append(row) + return { + 'available': available, + 'runs': rows, + 'checked_at': now.strftime(_SECONDS_FORMAT), + 'truncated': len(items) > limit, + } + + +__all__ = ( + 'CONVERSATION_LIMIT', + 'GLOBAL_LIMIT', + 'LIVE_READ_LIMIT', + 'WORKFLOW_READ_LIMIT', + 'RECENT_DELIVERY_SECONDS', + 'INVALID_CONVERSATION_CODE', + 'STATUS_UNAVAILABLE_CODE', + 'ROW_STATUSES', + 'ROW_PHASES', + 'DELIVERY_NOT_APPLICABLE', + 'DELIVERY_ROW_STATUSES', + 'DELIVERY_REASONS', + 'RETRY_BLOCKED_WORKFLOW_DELETED', + 'RETRY_BLOCKED_ALREADY_RUNNING', + 'RETRY_BLOCKED_DEFINITION_CHANGED', + 'RETRY_BLOCKED_NOT_RESUMABLE', + 'RETRY_BLOCKED_DEADLINE_EXCEEDED', + 'RETRY_BLOCKED_UNAVAILABLE', + 'RETRY_BLOCKED_CODES', + 'WAITING_ACTION_APPROVE', + 'WAITING_ACTION_RECONNECT', + 'WAITING_ACTION_OPEN_RUN', + 'CONVERSATION_QUERY', + 'GLOBAL_QUERY', + 'WorkflowRunStatusError', + 'WorkflowRunStatusServices', + 'default_workflow_run_status_services', + 'parse_status_conversation_id', + 'status_query', + 'workflow_run_status_payload', +) diff --git a/application/single_app/route_backend_orchestration.py b/application/single_app/route_backend_orchestration.py index ebc0e1c25..1b68c4725 100644 --- a/application/single_app/route_backend_orchestration.py +++ b/application/single_app/route_backend_orchestration.py @@ -207,7 +207,7 @@ normalize_elicitation, safe_failure, ) -from functions_settings import get_settings, get_user_settings +from functions_settings import enabled_required, get_settings, get_user_settings, workflow_user_required from functions_model_catalog import ModelCatalogError from functions_orchestration_model_routing import answer_selection, validate_auto_bindings from functions_prompt_metadata import build_prompt_selection_metadata @@ -1457,6 +1457,13 @@ def _workflow_run_links(): return functions_orchestration_workflow_run_links +def _workflow_run_status(): + """The workflow run status module, imported on first use like the run links module.""" + import functions_workflow_chat_delivery_status + + return functions_workflow_chat_delivery_status + + def _proposal_identity(user_id): """The requester's id, email, roles and tenant for a workflow proposal request. @@ -2930,3 +2937,40 @@ def orchestration_workflow_run_links(run_id): ) return jsonify(links.error_payload(links.SERVICE_UNAVAILABLE_CODE)), 503 return jsonify(payload), 200 + + @bp.route("/api/v2/orchestration/workflow-runs/status", methods=["GET"]) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required('allow_user_workflows') + @workflow_user_required + def orchestration_workflow_run_status(): + """Status rows for the saved workflow runs the requester's chats started. + + With one ``conversation_id``, the rows are that chat's runs, newest request first. Without + one, they are the requester's runs still in flight, still waiting to post back, or posted + in the last ten minutes. Each row reads only the requester's own runs and carries fixed + codes and labels, never a run's output or error text. Nothing is written. + """ + user_id = get_current_user_id() + if not user_id: + return jsonify({'error': 'User not authenticated'}), 401 + status = _workflow_run_status() + conversation_values = request.args.getlist('conversation_id') + try: + payload = status.workflow_run_status_payload(user_id, conversation_values) + except status.WorkflowRunStatusError as exc: + body, code = exc.payload() + return jsonify(body), code + except Exception as exc: + conversation_id = conversation_values[0] if len(conversation_values) == 1 else None + log_event( + '[ORCHESTRATION] The workflow run status could not be loaded.', level=logging.ERROR, + extra={ + **workflow_log_context(conversation_id=conversation_id), + 'stage': 'workflow_run_status', 'error_type': type(exc).__name__, + }, + ) + body, code = status.WorkflowRunStatusError(status.STATUS_UNAVAILABLE_CODE).payload() + return jsonify(body), code + return jsonify(payload), 200 diff --git a/functional_tests/route_tests/test_route_blueprint_policy_inventory.py b/functional_tests/route_tests/test_route_blueprint_policy_inventory.py index 687d326fa..07e584fe5 100644 --- a/functional_tests/route_tests/test_route_blueprint_policy_inventory.py +++ b/functional_tests/route_tests/test_route_blueprint_policy_inventory.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for route blueprint policy inventory. -Version: 0.261.214 +Version: 0.261.218 Implemented in: 0.242.069 Plan editor policy coverage: 0.261.102 Selected-group context policy coverage: 0.261.126 @@ -12,6 +12,7 @@ Workflow proposal policy coverage: 0.261.207 Workflow assistant policy coverage: 0.261.208 Workflow result context policy coverage: 0.261.214 +Workflow run status policy coverage: 0.261.218 This test ensures every SimpleChat route is assigned to a Blueprint-based security policy or an explicit reviewed route exemption. @@ -537,6 +538,29 @@ def test_workflow_run_link_route_keeps_the_orchestration_security_policy() -> No assert {"swagger_route", "login_required", "user_required"} <= set(route.decorator_names) +def test_workflow_run_status_route_keeps_the_personal_workflow_security_policy() -> None: + """The run status rows stay owner-only, behind the same gates as the personal runtime routes. + + Explicit raises keep this check under ``python -O``. + """ + path = "/api/v2/orchestration/workflow-runs/status" + routes = [ + route for route in iter_route_functions() + if route.file_name == "route_backend_orchestration.py" and route.path == path + ] + if len(routes) != 1: + raise AssertionError(f"Expected one status route, found {len(routes)}.") + route = routes[0] + if route.function_name != "orchestration_workflow_run_status" or route.route_target != "bp": + raise AssertionError(f"Unexpected status route registration: {route.function_name} on {route.route_target}.") + expected = ( + "bp.route", "swagger_route", "login_required", "user_required", + "enabled_required", "workflow_user_required", + ) + if route.decorator_names != expected: + raise AssertionError(f"Status route decorators changed: {route.decorator_names}.") + + def test_content_screening_routes_keep_authenticated_blueprint_guards() -> None: """Evidence and decisions never become public, even while enrollment is off.""" routes = [ @@ -571,6 +595,7 @@ def test_content_screening_routes_keep_authenticated_blueprint_guards() -> None: test_plan_editor_routes_keep_the_orchestration_security_policy, test_workflow_proposal_routes_keep_the_orchestration_security_policy, test_workflow_run_link_route_keeps_the_orchestration_security_policy, + test_workflow_run_status_route_keeps_the_personal_workflow_security_policy, test_content_screening_routes_keep_authenticated_blueprint_guards, ] results = [] diff --git a/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py b/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py index 05f8bdfe1..98e18b015 100644 --- a/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py +++ b/functional_tests/route_tests/test_route_unauthenticated_policy_contract.py @@ -2,9 +2,10 @@ # test_route_unauthenticated_policy_contract.py """ Functional test for route unauthenticated access policy contract. -Version: 0.261.214 +Version: 0.261.218 Implemented in: 0.242.069 Workflow result context coverage: 0.261.214 +Workflow run status coverage: 0.261.218 This test ensures every SimpleChat route has an explicit expected unauthenticated access behavior: public, browser-session authenticated, admin-only, or external @@ -276,6 +277,25 @@ def test_workflow_result_context_requires_a_signed_in_user_session() -> None: assert {"login_required", "user_required"} <= set(matches[0].decorator_names) +def test_workflow_run_status_requires_a_signed_in_user_session() -> None: + """The run card's status rows are never public or bearer-only. + + Explicit raises keep this check under ``python -O``. + """ + path = "/api/v2/orchestration/workflow-runs/status" + matches = [route for route in iter_route_functions() if route.path == path] + + if [route.function_name for route in matches] != ["orchestration_workflow_run_status"]: + raise AssertionError(f"Unexpected status route functions: {[route.function_name for route in matches]}.") + if expected_policy(path) != "session_user_401_or_redirect": + raise AssertionError(f"Status route policy changed: {expected_policy(path)}.") + required = {"login_required", "user_required", "enabled_required", "workflow_user_required"} + if not required <= set(matches[0].decorator_names): + raise AssertionError(f"Status route is missing guards: {sorted(required - set(matches[0].decorator_names))}.") + if "accesstoken_required" in matches[0].decorator_names: + raise AssertionError("Status route must not accept bearer tokens.") + + if __name__ == "__main__": tests = [ test_every_route_has_unauthenticated_access_policy, @@ -283,6 +303,7 @@ def test_workflow_result_context_requires_a_signed_in_user_session() -> None: test_external_routes_use_bearer_auth_decorator, test_sensitive_admin_routes_have_admin_or_specialized_route_decorator, test_workflow_result_context_requires_a_signed_in_user_session, + test_workflow_run_status_requires_a_signed_in_user_session, ] results = [] for test in tests: diff --git a/functional_tests/test_workflow_chat_delivery_status_route.py b/functional_tests/test_workflow_chat_delivery_status_route.py new file mode 100644 index 000000000..161c6217e --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_status_route.py @@ -0,0 +1,822 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_status_route.py +""" +Functional test for the workflow chat delivery status route. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures that the workflow-run status payload and Flask route stay owner-scoped, bounded +and safe: queries are single-partition projections, global and conversation modes include exactly +the intended runs, rows expose stable status/action/retry fields without leaking run output or +exception text, live and workflow reads observe their caps, runtime resume versions match the +durable store, and the authenticated route preserves the workflow feature gates and fixed errors. +""" + +import ast +import copy +import importlib +import json +import os +import re +import subprocess +import sys +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 +from flask import Blueprint, Flask # noqa: E402 +from werkzeug.test import Client # noqa: E402 +from werkzeug.wrappers import Response # noqa: E402 + +import functions_workflow_chat_delivery_status as status # noqa: E402 +from functions_m365_workflow_binding import M365_WAITING_STATES # noqa: E402 +from functions_workflow_alert_safety import sanitize_workflow_alert_record # noqa: E402 +from functions_workflow_chat_delivery import ( # noqa: E402 + CHAT_TRIGGER_SOURCE, + DELIVERY_MESSAGE_ID_PREFIX, + OPEN_STATUSES, + REASON_DEADLINE_EXCEEDED, + RUN_DOCUMENT_TERMINAL_STATUSES, + STATUS_DELIVERED, + STATUS_EXPIRED, + STATUS_READY, + format_delivery_timestamp, + parse_delivery_timestamp, +) +from functions_workflow_definitions import workflow_definition_for_editor, workflow_definition_revision # noqa: E402 +from functions_workflow_runtime_store import ( # noqa: E402 + WorkflowRuntimeConflict, + WorkflowRuntimeStore, + workflow_runtime_projection, +) +from test_orchestration_harness_routes import modules # noqa: F401, E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + NOW, + OTHER_USER, + REQUESTED_AT, + RUN_ID, + USER, + WORKFLOW_ID, + FakeClock, + FakeContainer, + cosmos_error, + iso, + make_invocation, + make_record, + make_run, + make_workflow, + parse_iso, + require, + shift, +) +from test_workflow_durable_execution import Clock, RuntimeContainer # noqa: E402 +from test_workflow_structured_flow import create_structured_runtime, definition # noqa: E402 + + +APP_ROOT = ROOT / "application" / "single_app" +STATUS_PATH = APP_ROOT / "functions_workflow_chat_delivery_status.py" +WORKFLOW_RUNS_PATH = APP_ROOT / "functions_orchestration_workflow_runs.py" +SETTINGS = { + "enable_chat_orchestration": True, + "allow_user_workflows": True, + "require_member_of_workflow_user": False, +} + +ALLOWED_PROJECTION_PATHS = { + "id", + "workflow_id", + "workflow_name", + "status", + "started_at", + "completed_at", + "progress", + "runtime_version", + "definition_revision", + "chat_invocation.conversation_id", + "chat_invocation.orchestration_run_id", + "chat_invocation.step_id", + "chat_invocation.requested_at", + "runtime.version", + "runtime.state", + "runtime.phase", + "runtime.progress", + "runtime.can_resume", + "runtime.deleted", + "runtime.limits.deadline_at", + "runtime.gate.id", + "runtime.gate.reason_code", + "chat_delivery.status", + "chat_delivery.generation", + "chat_delivery.message_id", + "chat_delivery.delivered_at", + "chat_delivery.outcome_reason", +} + + +class StrictQueryContainer(FakeContainer): + """FakeContainer that refuses cross-partition status queries.""" + + def query_items(self, query=None, parameters=None, partition_key=None, + enable_cross_partition_query=None, **kwargs): + require(enable_cross_partition_query is None, "status queries must not enable cross-partition reads") + return super().query_items( + query=query, + parameters=parameters, + partition_key=partition_key, + enable_cross_partition_query=enable_cross_partition_query, + **kwargs, + ) + + +class LiveRecorder: + """Recording live-status fake keyed by run id.""" + + def __init__(self, projections=None, raises=None): + self.projections = dict(projections or {}) + self.raises = set(raises or ()) + self.calls = [] + + def __call__(self, workflow, run_id, *, reader_user_id): + self.calls.append({"workflow": copy.deepcopy(workflow), "run_id": run_id, "reader_user_id": reader_user_id}) + if run_id in self.raises: + raise RuntimeError("secret-live-exception") + return copy.deepcopy(self.projections.get(run_id)) + + +def _path_value(document, dotted): + current = document + for part in dotted.split("."): + if not isinstance(current, dict) or part not in current: + return None, False + current = current[part] + return current, True + + +def _projection_pairs(query): + select = re.search(r"SELECT TOP \d+ (.*?) FROM c ", query) + require(select is not None, f"query must have a SELECT projection: {query!r}") + pairs = re.findall(r"c\.([A-Za-z0-9_.]+) AS ([A-Za-z0-9_]+)", select.group(1)) + require(pairs, "status query must project aliased paths") + return pairs + + +def _top(query): + match = re.search(r"SELECT TOP (\d+)", query) + require(match is not None, f"query must have TOP: {query!r}") + return int(match.group(1)) + + +def _project_run(document, pairs): + row = {} + for path, alias in pairs: + value, present = _path_value(document, path) + if present: + row[alias] = copy.deepcopy(value) + return row + + +def status_query_handler(container, query, params, partition_key): + require(partition_key == params.get("@user_id"), "status query must use the signed-in user's partition") + require("SELECT *" not in query, "status query must not use SELECT *") + pairs = _projection_pairs(query) + require({path for path, _alias in pairs} <= ALLOWED_PROJECTION_PATHS, "status projection includes an unsafe path") + rows = [] + for item in container.items.values(): + invocation = item.get("chat_invocation") + delivery = item.get("chat_delivery") if isinstance(item.get("chat_delivery"), dict) else {} + if item.get("user_id") != params.get("@user_id"): + continue + if item.get("trigger_source") != params.get("@source"): + continue + if not isinstance(invocation, dict): + continue + if not invocation.get("requested_at"): + continue + if "@conversation_id" in params: + if invocation.get("conversation_id") != params["@conversation_id"]: + continue + else: + terminal = item.get("status") in set(params.get("@terminal") or []) + open_delivery = delivery.get("status") in set(params.get("@open") or []) + recent_delivery = ( + isinstance(delivery.get("delivered_at"), str) + and delivery.get("delivered_at") >= params.get("@recent", "") + ) + if terminal and not open_delivery and not recent_delivery: + continue + rows.append(item) + rows.sort(key=lambda row: str((row.get("chat_invocation") or {}).get("requested_at") or ""), reverse=True) + return [_project_run(row, pairs) for row in rows[:_top(query)]] + + +def editor_workflow(workflow): + cleaned = {key: value for key, value in workflow.items() if not str(key).startswith("_")} + return workflow_definition_for_editor(sanitize_workflow_alert_record(cleaned)) + + +def with_revision(workflow): + value = copy.deepcopy(workflow) + value["definition_revision"] = workflow_definition_revision(editor_workflow(value)) + return value + + +def services_for(runs=None, workflows=None, *, live=None, settings=None, gate=None, clock=None): + runs_container = StrictQueryContainer("runs", runs or [], query_handler=status_query_handler) + workflow_container = FakeContainer( + "workflows", + [with_revision(make_workflow())] if workflows is None else workflows, + ) + live_reader = live if live is not None else LiveRecorder() + return SimpleNamespace( + runs=runs_container, + workflows=workflow_container, + live=live_reader, + services=status.WorkflowRunStatusServices( + runs=runs_container, + workflows=workflow_container, + get_settings=lambda: copy.deepcopy(settings if settings is not None else SETTINGS), + settings_gate=gate if gate is not None else (lambda value: None), + live_status=live_reader, + clock=clock or FakeClock(NOW), + ), + ) + + +def payload_for(runs, workflows=None, conversation_values=None, **kwargs): + harness = services_for(runs, workflows, **kwargs) + payload = status.workflow_run_status_payload(USER, conversation_values, services=harness.services) + return payload, harness + + +def row_for(run, workflows=None, conversation_values=None, **kwargs): + payload, harness = payload_for([run], workflows, conversation_values or [CONVERSATION_ID], **kwargs) + require(len(payload["runs"]) == 1, f"expected one status row, got {payload['runs']!r}") + return payload["runs"][0], harness + + +def run_doc(identifier, *, state="queued", workflow_id=WORKFLOW_ID, user_id=USER, + conversation_id=CONVERSATION_ID, requested_at=REQUESTED_AT, runtime=None, + delivery=None, status_value=None, runtime_version=4, **overrides): + value = make_run( + status=status_value or state, + workflow_id=workflow_id, + user_id=user_id, + id=identifier, + chat_invocation=make_invocation(conversation_id=conversation_id, requested_at=requested_at), + chat_delivery=make_record() if delivery is None else delivery, + runtime=runtime if runtime is not None else {"version": 4, "state": state, "can_resume": False}, + runtime_version=runtime_version, + started_at=shift(REQUESTED_AT, minutes=-5), + ) + value.update(overrides) + return value + + +def resumable_run(identifier=RUN_ID, *, workflow=None, runtime=None, delivery=None, **overrides): + wf = with_revision(workflow or make_workflow()) + view = runtime or {"version": 7, "state": "failed", "can_resume": True} + runtime_version = overrides.pop("runtime_version", view.get("version", 7)) + definition_revision = overrides.pop("definition_revision", workflow_definition_revision(editor_workflow(wf))) + return run_doc( + identifier, + state="failed", + runtime=view, + delivery=delivery if delivery is not None else make_record(status=STATUS_READY), + definition_revision=definition_revision, + runtime_version=runtime_version, + **overrides, + ), wf + + +def require_codes(response, expected_status, expected_code): + body = response.get_json() + require(response.status_code == expected_status, f"expected {expected_status}, got {response.status_code}: {body}") + require(body.get("code") == expected_code, f"expected code {expected_code}, got {body}") + return body + + +def test_query_shape_is_single_partition_and_projection_only_in_both_modes(): + for conversation_values, expected_top in (([CONVERSATION_ID], "TOP 21"), (None, "TOP 51")): + payload, harness = payload_for([run_doc(RUN_ID)], conversation_values=conversation_values) + require(payload["checked_at"] == "2026-05-04T15:00:00Z", "checked_at should use the request clock") + require(len(harness.runs.query_calls) == 1, "exactly one runs query should be issued") + call = harness.runs.query_calls[0] + query = call["query"] + params = {entry["name"]: entry["value"] for entry in call["parameters"]} + require(call["partition_key"] == USER, "status query must use the requester partition key") + require(call.get("enable_cross_partition_query") is None, "status query must not enable cross-partition") + require("SELECT *" not in query, "status query must not project the whole document") + require(expected_top in query, f"query must use {expected_top}: {query}") + require("ORDER BY c.chat_invocation.requested_at DESC" in query, "query must sort newest first") + require( + "c.user_id = @user_id AND c.trigger_source = @source AND IS_DEFINED(c.chat_invocation)" in query, + "query must keep the owner/source/invocation filter", + ) + require({path for path, _alias in _projection_pairs(query)} <= ALLOWED_PROJECTION_PATHS, "unsafe projection") + if conversation_values: + require(params.get("@conversation_id") == CONVERSATION_ID, "conversation query must bind conversation_id") + else: + expected_recent = (parse_iso(NOW) - timedelta(seconds=status.RECENT_DELIVERY_SECONDS)).strftime( + "%Y-%m-%dT%H:%M:%S.%fZ" + ) + require("canceled" in params["@terminal"] and "cancelled" in params["@terminal"], "terminal params need both spellings") + require(params["@open"] == sorted(OPEN_STATUSES), "open delivery statuses must be sorted") + require(params["@recent"] == expected_recent, "recent boundary must be exactly ten minutes") + + +def test_global_query_includes_only_active_open_or_recent_owner_runs(): + recent = format_delivery_timestamp(parse_iso(NOW) - timedelta(minutes=10)) + old = format_delivery_timestamp(parse_iso(NOW) - timedelta(minutes=11)) + runs = [ + run_doc("run-active", state="running", requested_at=shift(REQUESTED_AT, minutes=5)), + run_doc("run-open-delivery", state="failed", delivery=make_record(status=STATUS_READY), requested_at=shift(REQUESTED_AT, minutes=4)), + run_doc("run-recent", state="completed", delivery=make_record(status=STATUS_DELIVERED, delivered_at=recent), requested_at=shift(REQUESTED_AT, minutes=3)), + run_doc("run-old", state="completed", delivery=make_record(status=STATUS_DELIVERED, delivered_at=old), requested_at=shift(REQUESTED_AT, minutes=2)), + run_doc("run-undeliverable", state="failed", delivery=make_record(status="undeliverable"), requested_at=shift(REQUESTED_AT, minutes=1)), + run_doc("run-other-user", state="running", user_id=OTHER_USER, requested_at=shift(REQUESTED_AT, minutes=6)), + ] + payload, _harness = payload_for(runs) + ids = [row["run_id"] for row in payload["runs"]] + require(ids == ["run-active", "run-open-delivery", "run-recent"], f"unexpected global rows: {ids}") + + +@pytest.mark.parametrize("runtime_state,expected_status,expected_phase,waiting_reason,waiting_action", [ + ("queued", "queued", "running", None, None), + ("running", "running", "running", None, None), + ("completed", "completed", "finished", None, None), + ("completed_partial", "completed_partial", "finished", None, None), + ("failed", "failed", "failed", None, None), + ("invalid", "failed", "failed", None, None), + ("incomplete", "failed", "failed", None, None), + ("skipped", "failed", "failed", None, None), + ("cancelled", "cancelled", "cancelled", None, None), + ("canceled", "cancelled", "cancelled", None, None), + ("waiting_approval", "waiting", "needs_you", "approval", status.WAITING_ACTION_APPROVE), + ("awaiting_sign_in", "waiting", "needs_you", "microsoft_365_reconnect", status.WAITING_ACTION_RECONNECT), + (sorted(M365_WAITING_STATES - {"awaiting_sign_in"})[0], "waiting", "needs_you", "microsoft_365_approval", status.WAITING_ACTION_OPEN_RUN), + ("waiting_output", "waiting", "needs_you", "output_review", status.WAITING_ACTION_OPEN_RUN), + ("waiting_recovery", "running", "running", "recovery", status.WAITING_ACTION_OPEN_RUN), + ("paused", "waiting", "needs_you", "paused", status.WAITING_ACTION_OPEN_RUN), + ("cancelling", "running", "running", None, None), +]) +def test_status_phase_waiting_and_action_mappings(runtime_state, expected_status, expected_phase, + waiting_reason, waiting_action): + state_for_doc = runtime_state + runtime = { + "version": 3, + "state": runtime_state, + "can_resume": runtime_state in {"failed", "invalid", "incomplete"}, + "gate": {"id": "gate-1", "reason_code": None}, + "progress": {"completed": 2, "total": 5}, + } + if runtime_state in RUN_DOCUMENT_TERMINAL_STATUSES: + extra = {"completed_at": shift(REQUESTED_AT, minutes=8)} + else: + extra = {} + run = run_doc(RUN_ID, state=state_for_doc, runtime=runtime, **extra) + row, _harness = row_for(run) + require(row["status"] == expected_status, f"{runtime_state} status mismatch: {row}") + require(row["phase"] == expected_phase, f"{runtime_state} phase mismatch: {row}") + require(row["step_index"] == 2 and row["step_count"] == 5, "progress should map to step counts") + require(row["step_label"] is None, "step_label is intentionally null") + require(row["actions"]["open_run"] is True, "open_run is always available") + require(row["actions"]["cancel"] == (runtime_state not in RUN_DOCUMENT_TERMINAL_STATUSES and runtime_state != "cancelling"), "cancel rule mismatch") + if waiting_reason is None: + require(row["waiting"] is None, f"{runtime_state} should not expose waiting info") + require(row["actions"]["approve"] is False, "approve needs an approval wait") + else: + require(row["waiting"]["reason"] == waiting_reason, f"{runtime_state} waiting reason mismatch") + require(row["waiting"]["action"] == waiting_action, f"{runtime_state} waiting action mismatch") + require(row["waiting"]["gate_id"] == "gate-1", "waiting gate id should be carried") + require(row["actions"]["approve"] == (waiting_action == status.WAITING_ACTION_APPROVE), "approve rule mismatch") + + +def test_unknown_runtime_state_falls_back_to_stored_status_then_reads_as_running(): + fallback, _harness = row_for(run_doc( + RUN_ID, state="failed", runtime={"version": 3, "state": "future_state", "can_resume": False}, + )) + require( + fallback["status"] == "failed" and fallback["phase"] == "failed", + f"an unknown projection state should fall back to the stored run status: {fallback}", + ) + unknown, _harness = row_for(run_doc( + "run-unknown", state="future_state", runtime={"version": 3, "state": "future_state", "can_resume": True}, + )) + require( + unknown["status"] == "running" and unknown["phase"] == "running", + f"a state the status module doesn't know yet should read as running: {unknown}", + ) + require(unknown["actions"]["retry"] is False, "a state the module doesn't know is never retryable") + require(unknown["retry_blocked"] is None, "retry_blocked applies only to resumable states") + + +def test_expired_approval_cancel_delivery_error_and_secret_mapping(): + secret = "SECRET-RUN-OUTPUT-TEXT" + run = run_doc( + RUN_ID, + state="paused", + runtime={ + "version": 3, + "state": "paused", + "can_resume": False, + "phase": REASON_DEADLINE_EXCEEDED, + "gate": {"id": "gate-1", "reason_code": REASON_DEADLINE_EXCEEDED}, + }, + delivery=make_record( + status=STATUS_DELIVERED, + message_id=f"{DELIVERY_MESSAGE_ID_PREFIX}abc", + delivered_at=format_delivery_timestamp(parse_iso(NOW)), + outcome_reason=REASON_DEADLINE_EXCEEDED, + ), + error=secret, + output={"text": secret}, + ) + row, _harness = row_for(run) + encoded = json.dumps(row, sort_keys=True) + require(row["status"] == "expired" and row["phase"] == "failed", "paused deadline should show expired") + require(row["error_code"] == REASON_DEADLINE_EXCEEDED, "expired rows should use deadline_exceeded") + require(row["actions"]["cancel"] is False and row["actions"]["approve"] is False, "expired rows should not cancel or approve") + require(row["delivery"]["status"] == STATUS_DELIVERED, "delivered status should be preserved") + require(row["delivery"]["message_id"] == f"{DELIVERY_MESSAGE_ID_PREFIX}abc", "valid delivery message id should be exposed") + require(row["delivery"]["delivered_at"] == "2026-05-04T15:00:00Z", "delivered_at should be seconds precision") + require(row["delivery"]["reason"] == REASON_DEADLINE_EXCEEDED, "known delivery reason should be exposed") + require(secret not in encoded, "run output and raw error text must not appear in status payload") + + +def test_delivery_not_applicable_and_ready_mapping(): + missing_delivery, _harness = row_for(run_doc(RUN_ID, delivery={})) + ready_delivery, _harness = row_for(run_doc("run-ready", delivery=make_record(status=STATUS_READY))) + require(missing_delivery["delivery"] == { + "status": status.DELIVERY_NOT_APPLICABLE, + "generation": None, + "message_id": None, + "delivered_at": None, + "reason": None, + }, "missing delivery should be not_applicable") + require(ready_delivery["delivery"]["status"] == "pending", "ready delivery should display as pending") + + +def test_live_reads_are_capped_fallback_safe_and_use_editor_workflow(): + workflows = [with_revision(make_workflow(_etag="etag-secret", _ts=123, active_run_id=""))] + projections = { + "run-11": { + "version": 99, + "state": "waiting_approval", + "phase": "approval", + "can_resume": False, + "gate": {"id": "gate-live", "reason_code": "approval"}, + "progress": {"completed": 4, "total": 4}, + }, + } + live = LiveRecorder(projections, raises={"run-10"}) + runs = [ + run_doc(f"run-{index:02d}", state="running", requested_at=shift(REQUESTED_AT, minutes=index)) + for index in range(12) + ] + payload, _harness = payload_for(runs, workflows, live=live) + rows = {row["run_id"]: row for row in payload["runs"]} + require(len(live.calls) == status.LIVE_READ_LIMIT, "live reads should stop at the cap") + require(rows["run-11"]["live"] is True, "live override should mark the row live") + require(rows["run-11"]["status"] == "waiting", "live state should override stored state") + require(rows["run-11"]["runtime_version"] == 99, "live version should become runtime_version") + require(rows["run-10"]["live"] is False and rows["run-10"]["status"] == "running", "raising live read should fall back") + require(rows["run-00"]["live"] is False and rows["run-01"]["live"] is False, "rows beyond live cap should use stored values") + require(all(call["reader_user_id"] == USER for call in live.calls), "live reads need reader_user_id") + require(all("_etag" not in call["workflow"] and "_ts" not in call["workflow"] for call in live.calls), "live workflow must be editor form") + + +def test_workflow_reads_are_cached_and_capped_for_retry_projection(): + workflow = with_revision(make_workflow(active_run_id="")) + runs = [] + workflows = [] + for index in range(22): + workflow_id = f"workflow-{index:02d}" + wf = with_revision({**workflow, "id": workflow_id}) + workflows.append(wf) + run, _wf = resumable_run( + f"run-{index:02d}", + workflow=wf, + workflow_id=workflow_id, + requested_at=shift(REQUESTED_AT, minutes=index), + ) + runs.append(run) + # Add a second row for workflow-00 to prove point reads are cached per workflow id. + repeated = copy.deepcopy(runs[0]) + repeated["id"] = "run-repeat" + repeated["chat_invocation"]["requested_at"] = shift(REQUESTED_AT, minutes=30) + runs.insert(0, repeated) + payload, harness = payload_for(runs, workflows) + by_id = {row["run_id"]: row for row in payload["runs"]} + read_calls = [call for call in harness.workflows.calls if call[0] == "read_item"] + require(len(read_calls) == status.WORKFLOW_READ_LIMIT, f"workflow reads should stop at cap: {read_calls}") + require(sum(1 for call in read_calls if call[1] == "workflow-00") == 1, "workflow reads should be cached") + require(by_id["run-repeat"]["actions"]["retry"] is True, "cached workflow row should still be retryable") + require(by_id["run-02"]["retry_blocked"] == status.RETRY_BLOCKED_UNAVAILABLE, "21st workflow should be over cap") + require(by_id["run-01"]["retry_blocked"] == status.RETRY_BLOCKED_UNAVAILABLE, "22nd workflow should be over cap") + + +@pytest.mark.parametrize("case_name,runtime_changes,workflow_changes,run_changes,expected", [ + ("projection-deleted", {"deleted": True}, {}, {}, status.RETRY_BLOCKED_WORKFLOW_DELETED), + ("not-resumable", {"can_resume": False}, {}, {}, status.RETRY_BLOCKED_NOT_RESUMABLE), + ("deadline", {"limits": {"deadline_at": "2026-05-04T14:59:59.000000Z"}}, {}, {}, status.RETRY_BLOCKED_DEADLINE_EXCEEDED), + ("workflow-deleting", {}, {"deleting": True}, {}, status.RETRY_BLOCKED_WORKFLOW_DELETED), + ("revision-mismatch", {}, {"name": "Changed name"}, {}, status.RETRY_BLOCKED_DEFINITION_CHANGED), + ("already-running", {}, {"active_run_id": "other-run"}, {}, status.RETRY_BLOCKED_ALREADY_RUNNING), + ("missing-revision", {}, {}, {"definition_revision": ""}, status.RETRY_BLOCKED_UNAVAILABLE), + ("missing-version", {"version": None}, {}, {"runtime_version": None}, status.RETRY_BLOCKED_UNAVAILABLE), +]) +def test_retry_blocked_codes(case_name, runtime_changes, workflow_changes, run_changes, expected): + workflow_body = make_workflow(active_run_id="") + workflow_body.update(workflow_changes) + workflow = with_revision(workflow_body) + runtime = {"version": 7, "state": "failed", "can_resume": True} + runtime.update(runtime_changes) + run, _workflow = resumable_run(workflow=workflow, runtime=runtime, **run_changes) + if case_name == "revision-mismatch": + run["definition_revision"] = "old-revision" + row, _harness = row_for(run, [workflow]) + require(row["retry_blocked"] == expected, f"{case_name} retry block mismatch: {row}") + require(row["actions"]["retry"] is False, f"{case_name} should not be retryable") + + +def test_retry_blocked_workflow_read_errors_and_retry_ok(): + workflow = with_revision(make_workflow(active_run_id="")) + ok_run, _workflow = resumable_run(workflow=workflow) + ok_row, _harness = row_for(ok_run, [workflow]) + require(ok_row["retry_blocked"] is None and ok_row["actions"]["retry"] is True, "valid failed run should be retryable") + + gone_row, _harness = row_for(ok_run, []) + require(gone_row["retry_blocked"] == status.RETRY_BLOCKED_WORKFLOW_DELETED, "missing workflow should block as deleted") + + harness = services_for([ok_run], [workflow]) + harness.workflows.fail("read_item", cosmos_error(500, "secret-workflow-error")) + payload = status.workflow_run_status_payload(USER, [CONVERSATION_ID], services=harness.services) + require(payload["runs"][0]["retry_blocked"] == status.RETRY_BLOCKED_UNAVAILABLE, "read errors should be unavailable") + + +@pytest.mark.parametrize("terminal_state", ["skipped", "cancelled"]) +def test_skipped_and_cancelled_are_never_retryable_even_with_can_resume(terminal_state): + runtime = {"version": 8, "state": terminal_state, "can_resume": True} + run = run_doc(RUN_ID, state=terminal_state, runtime=runtime, runtime_version=8, completed_at=NOW) + row, _harness = row_for(run) + require(row["retry_blocked"] is None, "non-resumable terminal states should not compute retry_blocked") + require(row["actions"]["retry"] is False, f"{terminal_state} must not be retryable") + + +def schema_one_failed_control(): + workflow = with_revision({"id": WORKFLOW_ID, "user_id": USER, "durable_execution": True, "active_run_id": ""}) + container = RuntimeContainer() + clock = Clock() + store = WorkflowRuntimeStore(container, workflow, RUN_ID, clock=clock) + store.initialize( + snapshot_ref={"storage": "cosmos", "schema_version": 1, "sha256": "a" * 64, "size_bytes": 100, "chunk_count": 1}, + definition_revision=workflow_definition_revision(editor_workflow(workflow)), + actor_user_id=USER, + request_id="request-one", + ) + token = store.claim(owner_id="worker") + control = store.transition(token, state="failed") + return workflow, store, control, clock + + +def status_row_for_control(workflow, run_id, control, clock_value, *, user_id=USER, conversation_id=CONVERSATION_ID): + run = run_doc( + run_id, + state=control["state"], + workflow_id=workflow["id"], + user_id=user_id, + conversation_id=conversation_id, + runtime=workflow_runtime_projection(control), + runtime_version=control["version"], + definition_revision=workflow_definition_revision(editor_workflow(workflow)), + delivery=make_record(status=STATUS_READY), + ) + harness = services_for([run], [workflow], clock=lambda: clock_value) + payload = status.workflow_run_status_payload(user_id, [conversation_id], services=harness.services) + require(len(payload["runs"]) == 1, f"expected runtime row: {payload}") + return payload["runs"][0] + + +def test_schema_one_status_runtime_version_is_resume_expected_version(): + workflow, store, control, clock = schema_one_failed_control() + row = status_row_for_control(workflow, RUN_ID, control, clock()) + require(row["actions"]["retry"] is True, f"schema-1 failed control should be retryable: {row}") + stale = row["runtime_version"] - 1 + with pytest.raises(WorkflowRuntimeConflict): + store.resume(expected_version=stale, actor_user_id=USER, request_id="resume-stale") + resumed = store.resume(expected_version=row["runtime_version"], actor_user_id=USER, request_id="resume-ok") + require(resumed["state"] == "queued", "schema-1 resume should requeue the control") + + +def test_schema_two_status_runtime_version_is_resume_expected_version(monkeypatch): + workflow, store, _container, clock = create_structured_runtime(definition(), monkeypatch) + token = store.claim(owner_id="worker") + control = store.transition(token, state="failed") + row = status_row_for_control(workflow, "run", control, clock(), user_id="owner", conversation_id="conversation-schema-2") + require(row["actions"]["retry"] is True, f"schema-2 failed control should be retryable: {row}") + with pytest.raises(WorkflowRuntimeConflict): + store.resume(expected_version=row["runtime_version"] - 1, actor_user_id="owner", request_id="resume-stale") + resumed = store.resume(expected_version=row["runtime_version"], actor_user_id="owner", request_id="resume-ok") + require(resumed["state"] == "queued", "schema-2 resume should requeue the control") + + +def test_workflow_run_chat_invocation_writer_sets_requested_at(): + tree = ast.parse(WORKFLOW_RUNS_PATH.read_text(encoding="utf-8"), filename=str(WORKFLOW_RUNS_PATH)) + candidates = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + for keyword in node.keywords: + if keyword.arg == "chat_invocation": + candidates.append(keyword.value) + require(candidates, "expected a chat_invocation writer in functions_orchestration_workflow_runs.py") + for candidate in candidates: + require(isinstance(candidate, ast.Dict), "chat_invocation should be built as an explicit dict") + keys = {key.value for key in candidate.keys if isinstance(key, ast.Constant)} + require("requested_at" in keys, "chat_invocation writer must include requested_at") + + +def test_error_paths_are_fixed_and_non_leaking(): + for values in ((CONVERSATION_ID, "again"), (""), ("bad value!")): + with pytest.raises(status.WorkflowRunStatusError) as captured: + status.workflow_run_status_payload(USER, values if isinstance(values, tuple) else [values], services=services_for().services) + body, code = captured.value.payload() + require(code == 400 and body["code"] == status.INVALID_CONVERSATION_CODE, f"invalid conversation should be 400: {body}") + + harness = services_for([run_doc(RUN_ID)]) + harness.runs.fail("query_items", RuntimeError("secret-query-text")) + with pytest.raises(status.WorkflowRunStatusError) as query_error: + status.workflow_run_status_payload(USER, [CONVERSATION_ID], services=harness.services) + body, code = query_error.value.payload() + require(code == 503 and body["code"] == status.STATUS_UNAVAILABLE_CODE, "query failure should be unavailable") + require("secret-query-text" not in json.dumps(body), "query exception text must not leak") + + gate_harness = services_for([run_doc(RUN_ID)], gate=lambda _settings: (_ for _ in ()).throw(RuntimeError("secret-gate-text"))) + with pytest.raises(status.WorkflowRunStatusError) as gate_error: + status.workflow_run_status_payload(USER, [CONVERSATION_ID], services=gate_harness.services) + require("secret-gate-text" not in json.dumps(gate_error.value.payload()[0]), "gate exception text must not leak") + + body, code = status.WorkflowRunStatusError("new-future-code").payload() + require(code == 503 and body["code"] == status.STATUS_UNAVAILABLE_CODE, "unknown codes should map to unavailable") + + unavailable = services_for([run_doc(RUN_ID)], gate=lambda _settings: "disabled") + payload = status.workflow_run_status_payload(USER, [CONVERSATION_ID], services=unavailable.services) + require(payload["available"] is False and len(payload["runs"]) == 1, "settings gate reason should not hide rows") + + many = [run_doc(f"run-{index:02d}", requested_at=shift(REQUESTED_AT, minutes=index)) for index in range(status.CONVERSATION_LIMIT + 1)] + payload, _harness = payload_for(many, conversation_values=[CONVERSATION_ID]) + require(payload["truncated"] is True and len(payload["runs"]) == status.CONVERSATION_LIMIT, "conversation rows should truncate") + + bad = run_doc("bad-run") + bad["workflow_id"] = "" + payload, _harness = payload_for([bad], conversation_values=[CONVERSATION_ID]) + require(payload["runs"] == [], "bad stored identifiers should skip the row") + + with pytest.raises(status.WorkflowRunStatusError) as user_error: + status.workflow_run_status_payload("", [CONVERSATION_ID], services=services_for().services) + require(user_error.value.payload()[1] == 503, "invalid user_id should be unavailable") + + +@pytest.mark.parametrize("optimized", [False, True]) +def test_status_module_import_smoke_does_not_load_config_or_runtime(optimized): + code = ( + "import sys; " + "import functions_workflow_chat_delivery_status; " + "print('config=' + str('config' in sys.modules)); " + "print('runtime=' + str('functions_workflow_runtime' in sys.modules))" + ) + command = [sys.executable] + if optimized: + command.append("-O") + command.extend(["-c", code]) + environment = os.environ.copy() + environment["PYTHONPATH"] = f"{ROOT / 'application' / 'single_app'}{os.pathsep}{ROOT / 'functional_tests'}" + result = subprocess.run(command, text=True, capture_output=True, env=environment, check=False) + require(result.returncode == 0, f"import smoke failed: {result.stderr}") + require("config=False" in result.stdout, f"status import should not load config: {result.stdout}") + require("runtime=False" in result.stdout, f"status import should not load runtime: {result.stdout}") + + +def _log_recorder(logs, source): + def log_event(message, *args, **kwargs): + logs.append({ + "source": source, + "message": str(message), + "extra": copy.deepcopy(kwargs.get("extra") or {}), + "level": kwargs.get("level"), + }) + return log_event + + +@pytest.fixture +def route_harness(modules, monkeypatch): + status_module = importlib.import_module("functions_workflow_chat_delivery_status") + settings_module = importlib.import_module("functions_settings") + settings = copy.deepcopy(SETTINGS) + route_logs = [] + + def access_status(user_id): + return True, None + + monkeypatch.setattr(modules.auth, "check_user_access_status", access_status) + monkeypatch.setattr(modules.route, "get_settings", lambda: copy.deepcopy(settings)) + monkeypatch.setattr(settings_module, "get_settings", lambda: copy.deepcopy(settings)) + monkeypatch.setattr(modules.route, "log_event", _log_recorder(route_logs, "route_backend_orchestration")) + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="workflow-status-route-test") + blueprint = Blueprint("backend_orchestration", __name__) + blueprint.before_request(modules.auth.user_required_blueprint()) + modules.route.register_route_backend_orchestration(blueprint) + app.register_blueprint(blueprint) + return SimpleNamespace( + modules=modules, + status=status_module, + settings=settings, + logs=route_logs, + app=app, + client=Client(app, Response), + monkeypatch=monkeypatch, + ) + + +def login(route_harness, *, user_id=USER, roles=("User",)): + serializer = route_harness.app.session_interface.get_signing_serializer(route_harness.app) + cookie = serializer.dumps({ + "user": { + "oid": user_id, + "roles": list(roles), + "tid": "tenant-1", + "preferred_username": "owner@example.com", + }, + }) + route_harness.client.set_cookie(route_harness.app.config["SESSION_COOKIE_NAME"], cookie) + + +def status_get(route_harness, query=None): + return route_harness.client.get("/api/v2/orchestration/workflow-runs/status", query_string=query or {}) + + +def test_route_requires_a_signed_in_user(route_harness): + response = status_get(route_harness) + require(response.status_code in {302, 401}, f"unauthenticated API request should be rejected: {response.status_code}") + + +def test_route_preserves_workflow_feature_gates(route_harness): + login(route_harness) + route_harness.settings["allow_user_workflows"] = False + disabled = status_get(route_harness) + require(disabled.status_code == 400, f"allow_user_workflows false should return 400: {disabled.get_json()}") + require(disabled.get_json()["error"] == "Allow User Workflows is disabled.", "enabled_required message changed") + + route_harness.settings["allow_user_workflows"] = True + route_harness.settings["require_member_of_workflow_user"] = True + forbidden = status_get(route_harness) + require(forbidden.status_code == 403, f"missing WorkflowUser role should return 403: {forbidden.get_json()}") + + +def test_route_maps_status_errors_and_generic_errors_without_secret_text(route_harness, monkeypatch): + login(route_harness) + services = services_for([run_doc(RUN_ID)]).services + monkeypatch.setattr(route_harness.status, "default_workflow_run_status_services", lambda: services) + repeated = route_harness.client.get( + "/api/v2/orchestration/workflow-runs/status?conversation_id=one&conversation_id=two" + ) + require_codes(repeated, 400, status.INVALID_CONVERSATION_CODE) + + def boom(_user_id, _conversation_values): + raise RuntimeError("secret-text") + + monkeypatch.setattr(route_harness.status, "workflow_run_status_payload", boom) + failed = status_get(route_harness) + body_text = failed.get_data(as_text=True) + require_codes(failed, 503, status.STATUS_UNAVAILABLE_CODE) + require("secret-text" not in body_text, "generic route error must not leak exception text") + require(route_harness.logs, "generic route failure should be logged") + extra = route_harness.logs[-1]["extra"] + require(extra.get("stage") == "workflow_run_status", f"log stage mismatch: {extra}") + require(extra.get("error_type") == "RuntimeError", f"log error_type mismatch: {extra}") + require("secret-text" not in json.dumps(route_harness.logs, default=str), "logs must not contain exception text") + + +def test_route_returns_status_payload_body_shape(route_harness, monkeypatch): + login(route_harness) + services = services_for([run_doc(RUN_ID)]).services + monkeypatch.setattr(route_harness.status, "default_workflow_run_status_services", lambda: services) + response = status_get(route_harness, {"conversation_id": CONVERSATION_ID}) + body = response.get_json() + require(response.status_code == 200, f"status route should succeed: {body}") + require(set(body) == {"available", "runs", "checked_at", "truncated"}, f"top-level body shape changed: {body}") + require(body["available"] is True and body["truncated"] is False, "body flags should be present") + require(len(body["runs"]) == 1, f"expected one route row: {body}") + require({"workflow_id", "run_id", "status", "actions", "delivery"} <= set(body["runs"][0]), "row shape is incomplete") + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From 309e1c31f662c795cfeec6a6c1c62083d7cea0f8 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 03:27:40 -0400 Subject: [PATCH 10/36] Add workflow chat delivery control pin and notice/unread tests; require every id for guarded unread marks (6b-1) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_conversation_unread.py | 6 +- ...est_workflow_chat_delivery_control_pins.py | 408 ++++++++++++++++++ ...orkflow_chat_delivery_notice_and_unread.py | 330 ++++++++++++++ 3 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 functional_tests/test_workflow_chat_delivery_control_pins.py create mode 100644 functional_tests/test_workflow_chat_delivery_notice_and_unread.py diff --git a/application/single_app/functions_conversation_unread.py b/application/single_app/functions_conversation_unread.py index a643cbaa2..31d8d33e8 100644 --- a/application/single_app/functions_conversation_unread.py +++ b/application/single_app/functions_conversation_unread.py @@ -115,8 +115,10 @@ def mark_conversation_unread_guarded( cache after ``UNREAD_MARKED``. Storage errors other than a missing item or an ETag conflict propagate. """ - if not isinstance(message_id, str) or not message_id or not isinstance(planned_at, str) or not planned_at: - raise ValueError('An unread mark needs a message id and a planned time.') + # An empty user id must never match a conversation that has no owner, so every id is required. + required = (conversation_id, user_id, message_id, planned_at) + if not all(isinstance(value, str) and value for value in required): + raise ValueError('An unread mark needs a conversation id, a user id, a message id and a planned time.') attempts = max(1, int(max_retries) + 1) conversation_item = None for _attempt in range(attempts): diff --git a/functional_tests/test_workflow_chat_delivery_control_pins.py b/functional_tests/test_workflow_chat_delivery_control_pins.py new file mode 100644 index 000000000..e2b0e18cf --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_control_pins.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_control_pins.py +""" +Functional test for workflow chat delivery control generation pins. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test pins that terminal workflow runtime controls only advance delivery generation on resume +or tombstone, so chat delivery records reopen exactly when the durable control version really moves. +""" + +import copy +import inspect +import os +import sys +from datetime import timedelta +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +from functions_workflow_alert_safety import sanitize_workflow_alert_record # noqa: E402 +from functions_workflow_chat_delivery import ( # noqa: E402 + PHASE_MESSAGE_CREATED, + REASON_RUNTIME_MISSING, + REASON_WORKFLOW_DELETED, + STATUS_DELIVERED, + STATUS_PENDING, + STATUS_READY, + STATUS_UNDELIVERABLE, + control_summary, + reconcile_chat_delivery, +) +from functions_workflow_definitions import workflow_definition_for_editor, workflow_definition_revision # noqa: E402 +from functions_workflow_runtime_store import ( # noqa: E402 + WorkflowRuntimeConflict, + WorkflowRuntimeStore, + RuntimeUnavailable, + TERMINAL_STATES, +) +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + NOW, + RUN_ID, + USER, + WORKFLOW_ID, + make_record, + parse_iso, + require, +) +from test_workflow_durable_execution import Clock, RuntimeContainer # noqa: E402 +from test_workflow_structured_flow import create_structured_runtime, definition # noqa: E402 + + +READ_ONLY = { + "assert_owned", + "journal_page", + "journal_read", + "read", + "run_definition", +} +NON_RESUME_MUTATORS = { + "claim", + "decide", + "expire_deadline", + "heartbeat", + "initialize", + "journal_commit", + "journal_commit_many", + "journal_decide", + "journal_request", + "pause_execution_limit", + "release", + "requeue_m365", + "requeue_output", + "request_cancel", + "transition", + "update", + "wait", + "write_record", +} +VERSION_BUMPERS = {"resume", "tombstone"} + +TERMINAL_STATE_IDS = tuple(sorted(TERMINAL_STATES)) +SCHEMA_IDS = ("schema1", "schema2") +EXPECTED_REFUSALS = (WorkflowRuntimeConflict, RuntimeUnavailable, ValueError, PermissionError) + + +def editor_workflow(workflow): + cleaned = {key: value for key, value in workflow.items() if not str(key).startswith("_")} + return workflow_definition_for_editor(sanitize_workflow_alert_record(cleaned)) + + +def with_revision(workflow): + value = copy.deepcopy(workflow) + value["definition_revision"] = workflow_definition_revision(editor_workflow(value)) + return value + + +def schema_one_runtime(): + workflow = with_revision({"id": WORKFLOW_ID, "user_id": USER, "durable_execution": True, "active_run_id": ""}) + container = RuntimeContainer() + clock = Clock() + store = WorkflowRuntimeStore(container, workflow, RUN_ID, clock=clock) + store.initialize( + snapshot_ref={"storage": "cosmos", "schema_version": 1, "sha256": "a" * 64, "size_bytes": 100, "chunk_count": 1}, + definition_revision=workflow_definition_revision(editor_workflow(workflow)), + actor_user_id=USER, + request_id="request-one", + ) + return workflow, store, container, clock + + +def runtime_for_schema(schema_id, monkeypatch): + if schema_id == "schema1": + return schema_one_runtime() + return create_structured_runtime(definition(), monkeypatch) + + +def terminal_runtime(schema_id, terminal_state, monkeypatch, *, advance_past_deadline=True): + workflow, store, container, clock = runtime_for_schema(schema_id, monkeypatch) + token = store.claim(owner_id="worker", ttl_seconds=1) + require(token is not None, f"{schema_id}/{terminal_state}: expected the store to issue a lease") + control = store.transition(token, state=terminal_state) + require(control["state"] == terminal_state, f"{schema_id}/{terminal_state}: transition did not reach terminal state") + if advance_past_deadline: + clock.now += timedelta(days=3) + return workflow, store, container, clock, token, control + + +def snapshot(control): + return control["version"], control["state"], bool(control.get("deleted")) + + +def gate_id(control): + gate = control.get("gate") if isinstance(control.get("gate"), dict) else {} + return gate.get("id") or "gate-pin" + + +def initialization_args(control, *, request_id=None): + return { + "snapshot_ref": copy.deepcopy(control["snapshot_ref"]), + "definition_revision": copy.deepcopy(control["definition_revision"]), + "actor_user_id": control["actor_user_id"], + "request_id": request_id or control["request_id"], + } + + +def request_id(method, schema_id, state, index): + return f"pin-{schema_id}-{state}-{method}-{index}" + + +def non_resume_calls(store, token, control, schema_id, state): + current_version = control["version"] + gate = gate_id(control) + common_record = { + "id": f"pin-record-{schema_id}-{state}", + "run_id": store.identity["run_id"], + "payload": {"ok": True}, + } + waiting_gate = { + "id": "pin-wait-gate", + "kind": "approval", + "choices": ["approve", "reject"], + } + entries = [{ + "kind": "request", + "key": ["pin", schema_id, state], + "payload": {"action": "cancel", "actor_user_id": USER}, + "immutable": True, + }] + return [ + ("claim", lambda: store.claim(owner_id="worker-two", ttl_seconds=1)), + ("decide", lambda: store.decide( + expected_version=current_version, + gate_id=gate, + choice="approve", + actor_user_id=USER, + request_id=request_id("decide", schema_id, state, 1), + )), + ("expire_deadline", store.expire_deadline), + ("heartbeat", lambda: store.heartbeat(token, ttl_seconds=1)), + ("initialize-replay", lambda: store.initialize(**initialization_args(control))), + ("initialize-conflict", lambda: store.initialize( + **initialization_args(control, request_id=request_id("initialize", schema_id, state, 1)), + )), + ("journal_commit", lambda: store.journal_commit( + token, + "request", + ["direct-commit", schema_id, state], + {"action": "cancel", "actor_user_id": USER}, + immutable=True, + )), + ("journal_commit_many", lambda: store.journal_commit_many(token, entries)), + ("journal_decide", lambda: store.journal_decide( + expected_version=current_version, + gate_id=gate, + choice="approve", + actor_user_id=USER, + request_id=request_id("journal-decide", schema_id, state, 1), + )), + ("journal_request_cancel", lambda: store.journal_request( + "cancel", + actor_user_id=USER, + request_id=request_id("journal-cancel", schema_id, state, 1), + )), + ("journal_request_bad_action", lambda: store.journal_request( + "pause", + actor_user_id=USER, + request_id=request_id("journal-pause", schema_id, state, 1), + )), + ("pause_execution_limit", lambda: store.pause_execution_limit(token, "deadline_exceeded")), + ("release", lambda: store.release(token)), + ("requeue_m365", lambda: store.requeue_m365(expected_version=current_version, gate_id=gate)), + ("requeue_output", lambda: store.requeue_output(expected_version=current_version, gate_id=gate)), + ("request_cancel", lambda: store.request_cancel( + actor_user_id=USER, + request_id=request_id("cancel", schema_id, state, 1), + )), + ("transition", lambda: store.transition(token, state="failed")), + ("update", lambda: store.update(token, {"progress": {"phase": "pin"}}, expected_version=current_version)), + ("wait", lambda: store.wait(token, state="waiting_approval", gate=waiting_gate)), + ("write_record", lambda: store.write_record(token, common_record, immutable=True)), + ] + + +def require_control_unchanged(store, before, label, log): + after = store.read(allow_deleted=True) + if snapshot(after) != snapshot(before): + require( + False, + f"{label}: terminal control changed after non-resume call. " + f"Before={snapshot(before)} After={snapshot(after)} Log={log}", + ) + return after + + +def exercise_non_resume_mutators(store, token, schema_id, state): + before = store.read(allow_deleted=True) + log = [] + for index, (name, call) in enumerate(non_resume_calls(store, token, before, schema_id, state), start=1): + outcome = "returned" + try: + call() + except EXPECTED_REFUSALS as exc: + outcome = f"raised {type(exc).__name__}" + log.append(f"{schema_id}/{state}/{name}: {outcome}") + before = require_control_unchanged(store, before, f"{schema_id}/{state}/{name}", log) + return log + + +def delivered_record(generation): + return make_record( + status=STATUS_DELIVERED, + generation=generation, + phase=PHASE_MESSAGE_CREATED, + message_id=f"message-{generation}", + delivered_at=NOW, + ) + + +def ready_record(generation): + return make_record(status=STATUS_READY, generation=generation) + + +def reconcile(record, control): + return reconcile_chat_delivery(record, control_summary(control), now=parse_iso(NOW)) + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.218") + + +def test_store_public_methods_are_classified_for_terminal_generation_rules(): + public_methods = { + name for name, member in inspect.getmembers(WorkflowRuntimeStore, callable) + if not name.startswith("_") + } + classified = READ_ONLY | NON_RESUME_MUTATORS | VERSION_BUMPERS + missing = sorted(public_methods - classified) + unknown = sorted(classified - public_methods) + overlaps = sorted( + (READ_ONLY & NON_RESUME_MUTATORS) + | (READ_ONLY & VERSION_BUMPERS) + | (NON_RESUME_MUTATORS & VERSION_BUMPERS) + ) + for name in missing + unknown + overlaps: + require( + False, + f"Classify {name} and check the reopen rule in reconcile_chat_delivery " + "(functions_workflow_chat_delivery.py): a terminal control's version may only move on resume or tombstone.", + ) + + +@pytest.mark.parametrize("schema_id", SCHEMA_IDS) +@pytest.mark.parametrize("terminal_state", TERMINAL_STATE_IDS) +def test_non_resume_mutators_leave_terminal_controls_pinned(schema_id, terminal_state, monkeypatch): + _workflow, store, _container, _clock, token, _control = terminal_runtime(schema_id, terminal_state, monkeypatch) + exercise_non_resume_mutators(store, token, schema_id, terminal_state) + + +@pytest.mark.parametrize("schema_id", SCHEMA_IDS) +@pytest.mark.parametrize("terminal_state", ["completed", "failed"]) +def test_tombstone_bumps_and_then_non_resume_mutators_stay_pinned(schema_id, terminal_state, monkeypatch): + _workflow, store, _container, _clock, token, control = terminal_runtime(schema_id, terminal_state, monkeypatch) + before_version = control["version"] + tombstoned = store.tombstone() + require(tombstoned["version"] > before_version, f"{schema_id}/{terminal_state}: tombstone should bump version") + require(tombstoned.get("deleted") is True, f"{schema_id}/{terminal_state}: tombstone should set deleted") + require(tombstoned["state"] == "cancelled", f"{schema_id}/{terminal_state}: tombstone should cancel") + with pytest.raises(WorkflowRuntimeConflict): + store.read() + allowed = store.read(allow_deleted=True) + require(snapshot(allowed) == snapshot(tombstoned), f"{schema_id}/{terminal_state}: allow_deleted should read tombstone") + exercise_non_resume_mutators(store, token, schema_id, f"{terminal_state}-tombstoned") + + +@pytest.mark.parametrize("schema_id", SCHEMA_IDS) +@pytest.mark.parametrize("terminal_state", TERMINAL_STATE_IDS) +def test_resume_is_the_only_non_delete_terminal_version_bumper(schema_id, terminal_state, monkeypatch): + _workflow, store, _container, _clock, _token, control = terminal_runtime( + schema_id, + terminal_state, + monkeypatch, + advance_past_deadline=False, + ) + before = snapshot(control) + if terminal_state in {"failed", "invalid", "incomplete"}: + with pytest.raises(WorkflowRuntimeConflict): + store.resume(expected_version=control["version"] - 1, actor_user_id=USER, request_id="resume-stale") + stale_after = store.read(allow_deleted=True) + require(snapshot(stale_after) == before, f"{schema_id}/{terminal_state}: stale resume must not write") + resumed = store.resume(expected_version=control["version"], actor_user_id=USER, request_id="resume-ok") + require(resumed["version"] > control["version"], f"{schema_id}/{terminal_state}: resume should bump version") + require(resumed["state"] == "queued", f"{schema_id}/{terminal_state}: resume should queue") + require(resumed.get("deleted") is False, f"{schema_id}/{terminal_state}: resume should not delete") + else: + with pytest.raises(WorkflowRuntimeConflict): + store.resume(expected_version=control["version"], actor_user_id=USER, request_id="resume-refuse") + after = store.read(allow_deleted=True) + require(snapshot(after) == before, f"{schema_id}/{terminal_state}: non-resumable resume must not write") + + +def test_delivery_reconcile_reopens_only_for_real_resume_generations(monkeypatch): + _workflow, store, _container, _clock, token, control = terminal_runtime("schema1", "failed", monkeypatch) + generation = control["version"] + record = delivered_record(generation) + for line in exercise_non_resume_mutators(store, token, "schema1", "failed-delivery"): + updated, changed, _ready = reconcile(record, store.read(allow_deleted=True)) + require(not changed, f"{line}: delivered record should not change") + require(updated["status"] == STATUS_DELIVERED, f"{line}: delivered record should stay delivered") + + tombstoned = store.tombstone() + delivered_after_delete, changed, _ready = reconcile(record, tombstoned) + require(not changed, "deleted controls must not reopen delivered records") + require(delivered_after_delete["status"] == STATUS_DELIVERED, "delivered records stay delivered after delete") + closed, changed, _ready = reconcile(ready_record(generation), tombstoned) + require(changed, "ready record should close when workflow is deleted") + require(closed["status"] == STATUS_UNDELIVERABLE, "deleted workflow should be undeliverable") + require(closed["outcome_reason"] == REASON_WORKFLOW_DELETED, "deleted workflow reason should be pinned") + + _workflow, resume_store, _container, _clock, _token, failed = terminal_runtime( + "schema2", + "failed", + monkeypatch, + advance_past_deadline=False, + ) + resumed = resume_store.resume(expected_version=failed["version"], actor_user_id=USER, request_id="resume-delivery") + reopened, changed, _ready = reconcile(delivered_record(failed["version"]), resumed) + require(changed, "resume should reopen a delivered generation") + require(reopened["status"] == STATUS_PENDING, "resume should reset delivery to pending") + require(reopened["generation"] is None, "resume should clear current generation") + require(reopened["history"][-1]["generation"] == failed["version"], "resume should retain prior generation in history") + + missing_summary = control_summary(None) + missing_closed, changed, _ready = reconcile_chat_delivery(ready_record(7), missing_summary, now=parse_iso(NOW)) + require(changed, "missing control should close a ready record") + require(missing_closed["outcome_reason"] == REASON_RUNTIME_MISSING, "missing control reason should be pinned") + second, changed, _ready = reconcile_chat_delivery(missing_closed, missing_summary, now=parse_iso(NOW)) + require(not changed, "missing control closure should be stable on a second reconcile") + require(second["status"] == STATUS_UNDELIVERABLE, "missing control closure should stay undeliverable") + + deleted_summary = { + "exists": True, + "deleted": True, + "version": generation + 1, + "state": "cancelled", + "phase": None, + "gate_reason_code": None, + "deadline_at": None, + "deadline_seconds": None, + "schema_version": 1, + } + guarded, changed, _ready = reconcile_chat_delivery(delivered_record(generation), deleted_summary, now=parse_iso(NOW)) + require(not changed, "deleted summary with newer version must not reopen") + require(guarded["status"] == STATUS_DELIVERED, "deleted summary should leave delivered status") + same_version = control_summary(control) + guarded, changed, _ready = reconcile_chat_delivery(delivered_record(generation), same_version, now=parse_iso(NOW)) + require(not changed, "same generation must not reopen") + require(guarded["status"] == STATUS_DELIVERED, "same generation should leave delivered status") + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) diff --git a/functional_tests/test_workflow_chat_delivery_notice_and_unread.py b/functional_tests/test_workflow_chat_delivery_notice_and_unread.py new file mode 100644 index 000000000..b5ee79c42 --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_notice_and_unread.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_notice_and_unread.py +""" +Functional test for workflow chat delivery notices and unread marks. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures chat-started workflow delivery notifications stay idempotent and guarded unread marks preserve owner-scoped conversation state. +""" + +import copy +import importlib +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +from functions_conversation_unread import ( # noqa: E402 + UNREAD_ALREADY_MARKED, + UNREAD_CONFLICT, + UNREAD_MARKED, + UNREAD_NOT_FOUND, + UNREAD_READ_SINCE, + UNREAD_UNAVAILABLE, + mark_conversation_unread_guarded, +) +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + OTHER_USER, + REQUESTED_AT, + USER, + FakeContainer, + cosmos_error, + make_conversation, + require, + shift, +) + + +MESSAGE_ID = "assistant-workflow-result-1" +PLANNED_AT = shift(REQUESTED_AT, minutes=10) + + +class FakeConfigCosmosDatabase: + """Minimal database stand-in for config.py import-time container setup.""" + + def __init__(self): + self.containers = {} + + def create_container_if_not_exists(self, id, **kwargs): + if id not in self.containers: + self.containers[id] = FakeContainer(id) + return self.containers[id] + + +class FakeConfigCosmosClient: + """Minimal Cosmos client stand-in that never opens a network connection.""" + + def __init__(self, *args, **kwargs): + self.database = FakeConfigCosmosDatabase() + + def create_database_if_not_exists(self, *args, **kwargs): + return self.database + + +MODULES_WITH_CONFIG = ("config", "functions_notifications") + + +def import_app_module_without_live_cosmos(module_name): + """Import an app module while config.py sees only fake Cosmos clients.""" + if module_name in sys.modules: + return sys.modules[module_name] + + import azure.cosmos as azure_cosmos + + original_cosmos_client = azure_cosmos.CosmosClient + azure_cosmos.CosmosClient = FakeConfigCosmosClient + try: + return importlib.import_module(module_name) + finally: + azure_cosmos.CosmosClient = original_cosmos_client + + +@pytest.fixture(name="notifications") +def notifications_fixture(): + return import_app_module_without_live_cosmos("functions_notifications") + + +def test_version_is_at_least_the_chat_delivery_release(): + checked = assert_app_version_at_least("0.261.218") + require(isinstance(checked, str), f"version helper should return the current version, got {checked!r}") + + +def test_notification_type_matches_contract_and_registry(notifications): + delivery = importlib.import_module("functions_workflow_chat_delivery") + notice_type = notifications.WORKFLOW_CHAT_DELIVERY_NOTIFICATION_TYPE + contract_type = delivery.NOTIFICATION_TYPE + require(notice_type == "workflow_chat_delivery", f"notification constant changed: {notice_type}") + require(contract_type == notice_type, f"delivery contract type drifted: {contract_type}") + + entry = notifications.NOTIFICATION_TYPES.get(notice_type) + require(isinstance(entry, dict), "workflow chat delivery notification type must be registered") + icon = entry.get("icon") + color = entry.get("color") + require(icon == "bi-activity", f"workflow chat delivery icon changed: {entry}") + require(color == "info", f"workflow chat delivery color changed: {entry}") + + +def test_chat_response_notification_passes_retry_options_to_create_notification(notifications, monkeypatch): + captured = [] + + def fake_create_notification(**kwargs): + captured.append(copy.deepcopy(kwargs)) + return {"id": f"notice-{len(captured)}", **kwargs} + + monkeypatch.setattr(notifications, "create_notification", fake_create_notification) + first = notifications.create_chat_response_notification( + USER, + CONVERSATION_ID, + MESSAGE_ID, + conversation_title="Digest", + response_preview="Done", + idempotency_key="k-1", + strict=True, + ) + first_call = captured[-1] + require(first["id"] == "notice-1", f"patched notification helper returned unexpected doc: {first}") + require(first_call.get("idempotency_key") == "k-1", f"idempotency key was not passed through: {first_call}") + require(first_call.get("strict") is True, f"strict=True was not passed through: {first_call}") + + notifications.create_chat_response_notification(USER, CONVERSATION_ID, MESSAGE_ID) + default_call = captured[-1] + require(default_call.get("idempotency_key") is None, f"default idempotency key changed: {default_call}") + require(default_call.get("strict") is False, f"default strict flag changed: {default_call}") + + +def test_chat_response_notification_idempotency_returns_existing_notice(notifications, monkeypatch): + container = FakeContainer("notifications") + monkeypatch.setattr(notifications, "cosmos_notifications_container", container) + + first = notifications.create_chat_response_notification( + USER, CONVERSATION_ID, MESSAGE_ID, idempotency_key="same-key", strict=True, + ) + second = notifications.create_chat_response_notification( + USER, CONVERSATION_ID, MESSAGE_ID, idempotency_key="same-key", strict=True, + ) + stored_count = len(container.items) + require(stored_count == 1, f"same retry key should store exactly one notice, got {stored_count}") + require(second.get("id") == first.get("id"), f"repeat retry should return the existing notice: {first} {second}") + + third = notifications.create_chat_response_notification( + USER, CONVERSATION_ID, MESSAGE_ID, idempotency_key="different-key", strict=True, + ) + stored_count = len(container.items) + require(stored_count == 2, f"different retry key should create a second notice, got {stored_count}") + require(third.get("id") != first.get("id"), "different retry keys must not collide") + + +def test_chat_response_notification_strict_create_failure_contract(notifications, monkeypatch): + soft_container = FakeContainer("notifications-soft") + soft_container.fail("create_item", cosmos_error(500, "create failed")) + monkeypatch.setattr(notifications, "cosmos_notifications_container", soft_container) + soft_result = notifications.create_chat_response_notification( + USER, CONVERSATION_ID, MESSAGE_ID, idempotency_key="soft-failure", strict=False, + ) + require(soft_result is None, f"strict=False should swallow create failures and return None: {soft_result}") + + hard_container = FakeContainer("notifications-hard") + hard_container.fail("create_item", cosmos_error(500, "create failed")) + monkeypatch.setattr(notifications, "cosmos_notifications_container", hard_container) + with pytest.raises(Exception) as raised: + notifications.create_chat_response_notification( + USER, CONVERSATION_ID, MESSAGE_ID, idempotency_key="hard-failure", strict=True, + ) + status_code = getattr(raised.value, "status_code", None) + require(status_code == 500, f"strict=True should propagate the storage error, got {raised.value!r}") + + +def replace_count(container): + writes = container.writes("replace_item") + return len(writes) + + +def test_guarded_unread_marked_sets_fields_and_raises_last_updated(): + conversation = make_conversation(last_updated=shift(PLANNED_AT, minutes=-5)) + container = FakeContainer("conversations", [conversation]) + outcome, returned = mark_conversation_unread_guarded(container, CONVERSATION_ID, USER, MESSAGE_ID, PLANNED_AT) + stored = container.get(CONVERSATION_ID) + + require(outcome == UNREAD_MARKED, f"expected marked outcome, got {outcome}") + require(returned.get("id") == CONVERSATION_ID, f"returned conversation mismatch: {returned}") + require(stored.get("has_unread_assistant_response") is True, f"unread flag not set: {stored}") + require(stored.get("last_unread_assistant_message_id") == MESSAGE_ID, f"unread message mismatch: {stored}") + require(stored.get("last_unread_assistant_at") == PLANNED_AT, f"unread timestamp mismatch: {stored}") + require(stored.get("last_updated") == PLANNED_AT, f"last_updated should be raised to planned_at: {stored}") + + newer_last_updated = shift(PLANNED_AT, minutes=5) + newer_container = FakeContainer("conversations-newer", [make_conversation(last_updated=newer_last_updated)]) + newer_outcome, _newer_returned = mark_conversation_unread_guarded( + newer_container, CONVERSATION_ID, USER, "assistant-workflow-result-2", PLANNED_AT, + ) + newer_stored = newer_container.get(CONVERSATION_ID) + require(newer_outcome == UNREAD_MARKED, f"expected newer conversation to be marked: {newer_outcome}") + require(newer_stored.get("last_updated") == newer_last_updated, f"last_updated must not be lowered: {newer_stored}") + + +def test_guarded_unread_already_marked_does_not_write_again(): + container = FakeContainer("conversations", [make_conversation()]) + first_outcome, _first = mark_conversation_unread_guarded(container, CONVERSATION_ID, USER, MESSAGE_ID, PLANNED_AT) + writes_after_first = replace_count(container) + second_outcome, _second = mark_conversation_unread_guarded(container, CONVERSATION_ID, USER, MESSAGE_ID, PLANNED_AT) + writes_after_second = replace_count(container) + + require(first_outcome == UNREAD_MARKED, f"setup mark failed: {first_outcome}") + require(second_outcome == UNREAD_ALREADY_MARKED, f"expected already_marked, got {second_outcome}") + require(writes_after_second == writes_after_first, "already_marked should not issue another replace") + + +def test_guarded_unread_read_since_skips_without_writing(): + read_at = shift(PLANNED_AT, minutes=1) + container = FakeContainer("conversations", [make_conversation(last_updated=read_at)]) + outcome, conversation = mark_conversation_unread_guarded( + container, + CONVERSATION_ID, + USER, + MESSAGE_ID, + PLANNED_AT, + skip_if_read_since=PLANNED_AT, + ) + writes = replace_count(container) + require(outcome == UNREAD_READ_SINCE, f"expected read_since when last_updated reached skip time: {outcome}") + require(conversation.get("last_updated") == read_at, f"read_since should return the stored conversation: {conversation}") + require(writes == 0, f"read_since should not write, got {writes} replaces") + + +def test_guarded_unread_missing_conversation_returns_not_found(): + container = FakeContainer("conversations") + outcome, conversation = mark_conversation_unread_guarded(container, CONVERSATION_ID, USER, MESSAGE_ID, PLANNED_AT) + require(outcome == UNREAD_NOT_FOUND, f"missing conversation should return not_found, got {outcome}") + require(conversation is None, f"missing conversation should not return a document: {conversation}") + + +@pytest.mark.parametrize("case_id,overrides", [ + ("non_private", {"chat_type": "personal_multi_user"}), + ("soft_deleted", {"deleted": True}), + ("orchestration_deleted", {"orchestration_deleted": True}), + ("wrong_owner", {"user_id": OTHER_USER}), +]) +def test_guarded_unread_unavailable_cases_do_not_write(case_id, overrides): + container = FakeContainer("conversations", [make_conversation(**overrides)]) + outcome, conversation = mark_conversation_unread_guarded(container, CONVERSATION_ID, USER, MESSAGE_ID, PLANNED_AT) + writes = replace_count(container) + require(outcome == UNREAD_UNAVAILABLE, f"{case_id} should be unavailable, got {outcome}") + require(isinstance(conversation, dict), f"{case_id} should return the read conversation") + require(writes == 0, f"{case_id} should not write, got {writes} replaces") + + +def test_guarded_unread_require_private_false_allows_non_private_conversation(): + container = FakeContainer("conversations", [make_conversation(chat_type="personal_multi_user")]) + outcome, _conversation = mark_conversation_unread_guarded( + container, + CONVERSATION_ID, + USER, + MESSAGE_ID, + PLANNED_AT, + require_private=False, + ) + stored = container.get(CONVERSATION_ID) + require(outcome == UNREAD_MARKED, f"require_private=False should allow non-private conversations: {outcome}") + require(stored.get("last_unread_assistant_message_id") == MESSAGE_ID, f"non-private mark missing: {stored}") + + +def test_guarded_unread_conflict_after_retry_budget_counts_attempts(): + max_retries = 2 + expected_attempts = max_retries + 1 + container = FakeContainer("conversations", [make_conversation()]) + container.fail("replace_item", *[cosmos_error(412, "etag conflict") for _index in range(expected_attempts)]) + outcome, conversation = mark_conversation_unread_guarded( + container, CONVERSATION_ID, USER, MESSAGE_ID, PLANNED_AT, max_retries=max_retries, + ) + attempts = replace_count(container) + require(outcome == UNREAD_CONFLICT, f"exhausted 412 retries should return conflict, got {outcome}") + require(conversation.get("id") == CONVERSATION_ID, f"conflict should return the last read conversation: {conversation}") + require(attempts == expected_attempts, f"max_retries={max_retries} should make {expected_attempts} attempts, got {attempts}") + + +def test_guarded_unread_retry_preserves_concurrent_edit(): + container = FakeContainer("conversations", [make_conversation(title="Original title")]) + + def concurrent_title_update(): + current = container.get(CONVERSATION_ID) + current["title"] = "Concurrent title" + container.put(current) + return cosmos_error(412, "etag conflict") + + container.fail("replace_item", concurrent_title_update) + outcome, _conversation = mark_conversation_unread_guarded(container, CONVERSATION_ID, USER, MESSAGE_ID, PLANNED_AT) + stored = container.get(CONVERSATION_ID) + attempts = replace_count(container) + + require(outcome == UNREAD_MARKED, f"retry after one 412 should mark, got {outcome}") + require(attempts == 2, f"one conflict should produce exactly two replace attempts, got {attempts}") + require(stored.get("title") == "Concurrent title", f"retry should preserve concurrent title: {stored}") + require(stored.get("last_unread_assistant_message_id") == MESSAGE_ID, f"retry should still set unread mark: {stored}") + + +@pytest.mark.parametrize("field_name,conversation_id,user_id,message_id", [ + ("conversation_id_none", None, USER, MESSAGE_ID), + ("conversation_id_empty", "", USER, MESSAGE_ID), + ("user_id_none", CONVERSATION_ID, None, MESSAGE_ID), + ("user_id_empty", CONVERSATION_ID, "", MESSAGE_ID), + ("message_id_none", CONVERSATION_ID, USER, None), + ("message_id_empty", CONVERSATION_ID, USER, ""), +]) +def test_guarded_unread_rejects_missing_required_ids(field_name, conversation_id, user_id, message_id): + container = FakeContainer("conversations", [make_conversation()]) + with pytest.raises(ValueError, match="unread mark"): + mark_conversation_unread_guarded(container, conversation_id, user_id, message_id, PLANNED_AT) + writes = replace_count(container) + require(writes == 0, f"{field_name} should be rejected before writing, got {writes} replaces") + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From 74f420870be0f05d07751d97d64ae164e1724949 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 03:41:53 -0400 Subject: [PATCH 11/36] Check for the delivery message before compose at every phase; add save-guard tests (6b-1) A record reverted by a stale save (phase none) skipped the point read of the deterministic message and re-ran compose, the unread mark and the notice call. The worker now point-reads the message whenever the phase is below message_created, and exhaustion checks it at any phase. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...functions_workflow_chat_delivery_worker.py | 11 +- .../test_workflow_chat_delivery_save_guard.py | 398 ++++++++++++++++++ 2 files changed, 403 insertions(+), 6 deletions(-) create mode 100644 functional_tests/test_workflow_chat_delivery_save_guard.py diff --git a/application/single_app/functions_workflow_chat_delivery_worker.py b/application/single_app/functions_workflow_chat_delivery_worker.py index d19f1552a..771065885 100644 --- a/application/single_app/functions_workflow_chat_delivery_worker.py +++ b/application/single_app/functions_workflow_chat_delivery_worker.py @@ -1451,7 +1451,7 @@ def _exhausted(delivery, reason): """Attempts ran out. Close with a best-effort notice; never post a second message.""" phase = delivery.record.get('phase') created = phase_at_least(phase, PHASE_MESSAGE_CREATED) - if not created and phase_at_least(phase, PHASE_PUBLISHING): + if not created: try: created = _message_exists(delivery, delivery.message_id()) except _Transient: @@ -1552,11 +1552,10 @@ def _deliver(delivery): return _close_silently(delivery, REASON_WORKFLOW_DELETED) phase = delivery.record.get('phase') - if ( - phase_at_least(phase, PHASE_PUBLISHING) and not phase_at_least(phase, PHASE_MESSAGE_CREATED) - and _message_exists(delivery, delivery.message_id()) - ): - # The create landed but its phase write didn't. + if not phase_at_least(phase, PHASE_MESSAGE_CREATED) and _message_exists(delivery, delivery.message_id()): + # Check before compose (D9 step 4): an earlier attempt at this generation already posted M, + # whether its phase write was lost or the record itself was reverted. Skip compose and the + # unread mark; the notice below is idempotent by its key. _write_phase(delivery, PHASE_MESSAGE_CREATED) if phase_at_least(delivery.record.get('phase'), PHASE_MESSAGE_CREATED): return _notify_delivered(delivery) diff --git a/functional_tests/test_workflow_chat_delivery_save_guard.py b/functional_tests/test_workflow_chat_delivery_save_guard.py new file mode 100644 index 000000000..15b820246 --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_save_guard.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_save_guard.py +""" +Functional test for the workflow chat delivery guarded run save. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures chat-started workflow run saves preserve delivered chat-delivery state across stale full-document writes and worker retries. +""" + +import copy +import importlib +import logging +import sys +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 +from azure.cosmos.exceptions import CosmosHttpResponseError # noqa: E402 + +import functions_workflow_chat_delivery_worker as worker # noqa: E402 +from functions_conversation_unread import clear_conversation_unread # noqa: E402 +from functions_workflow_chat_delivery import ( # noqa: E402 + CHAT_TRIGGER_SOURCE, + PHASE_MESSAGE_CREATED, + STATUS_DELIVERED, + merge_stored_run_fields, +) +from test_orchestration_harness_routes import modules # noqa: F401, E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + RUN_ID, + USER, + FakeContainer, + cosmos_error, + make_invocation, + make_record, + make_run, + make_world, + require, + shift, +) + + +M5 = f"assistant_workflow_delivery_{RUN_ID}_{CONVERSATION_ID}_5" +COSMOS_SYSTEM_FIELDS = ("_rid", "_self", "_etag", "_attachments", "_ts") + + +@pytest.fixture +def personal_workflows(modules, monkeypatch): + module = importlib.import_module("functions_personal_workflows") + logs = [] + + def delivery_log(message, **kwargs): + logs.append({"message": message, **kwargs}) + + monkeypatch.setattr(module, "delivery_log", delivery_log) + return SimpleNamespace(module=module, logs=logs) + + +def deliver(world, run_id=RUN_ID, user_id=USER): + return worker.process_workflow_chat_delivery(user_id, run_id, services=world.services) + + +def sweep(world): + return worker.run_workflow_chat_delivery_sweep(world.services) + + +def make_due(world, run_id=RUN_ID): + world.clock.now = shift(world.record(run_id)["next_attempt_at"], seconds=1) + + +def user_reads_chat(world): + conversation = clear_conversation_unread(world.conversation()) + world.conversations.put(conversation) + + +def install_run_container(monkeypatch, personal_workflows, container): + monkeypatch.setattr( + personal_workflows.module, + "cosmos_personal_workflow_runs_container", + container, + ) + + +def save_run(personal_workflows, run_record, user_id=USER): + return personal_workflows.module.save_personal_workflow_run(user_id, copy.deepcopy(run_record)) + + +def delivery_counts(world): + calls = world.notifications.chat_calls + world.notifications.notice_calls + return { + "messages": len(world.delivery_messages()), + # Each idempotency key stores one notice, like the real helpers: a repeated call with the + # same key returns the stored notice instead of ringing the bell again. + "notices": len(world.notifications.notices()), + "notice_keys": len({call["idempotency_key"] for call in calls}), + "unread": len(world.unread_calls), + "model_requests": len(world.models.requests), + } + + +def assert_exactly_once(world, *, model_requests=1, unread=1): + counts = delivery_counts(world) + require(counts["messages"] == 1, f"expected one delivery message, got {counts}") + require(counts["notices"] == 1, f"expected one bell notice, got {counts}") + require(counts["notice_keys"] == 1, f"every notice call must reuse the generation's key, got {counts}") + require(counts["unread"] == unread, f"expected {unread} unread mark(s), got {counts}") + require(counts["model_requests"] == model_requests, f"expected {model_requests} model call(s), got {counts}") + + +def delivered_world_with_stale_copy(): + world = make_world() + stale = copy.deepcopy(world.run()) + outcome = deliver(world) + require(outcome == worker.OUTCOME_DELIVERED, f"initial delivery failed with {outcome}") + assert_exactly_once(world) + return world, stale + + +def reset_model_observations(world): + world.models.selected.requests.clear() + world.models.default.requests.clear() + world.models.calls.clear() + + +def reverted_pending_copy(stale): + reverted = copy.deepcopy(stale) + reverted["status"] = "completed" + return reverted + + +def replace_calls(container): + return container.writes("replace_item") + + +def upsert_calls(container): + return container.writes("upsert_item") + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.218") + + +def test_guarded_stale_save_preserves_delivery_and_sweep_stays_exactly_once(personal_workflows, monkeypatch): + world, stale = delivered_world_with_stale_copy() + install_run_container(monkeypatch, personal_workflows, world.runs) + delivered = copy.deepcopy(world.record()) + stale["progress"] = {"step": "runner-finished-after-delivery"} + stale["status"] = "completed_with_progress" + reset_model_observations(world) + + saved = save_run(personal_workflows, stale) + + stored = world.run() + require(saved["chat_delivery"] == delivered, "guarded save must return the stored delivered record") + require(stored["chat_delivery"] == delivered, "guarded save must preserve delivered chat_delivery") + require(stored["progress"] == {"step": "runner-finished-after-delivery"}, "incoming progress must be applied") + require(stored["status"] == "completed_with_progress", "incoming non-delivery status must be applied") + require(len(replace_calls(world.runs)) >= 1, "guarded save must use an ETag replace") + require(upsert_calls(world.runs) == [], "guarded stale save must not blind upsert when replace succeeds") + + summary = sweep(world) + + require(summary["processed"] == 0, f"delivered record should not sweep again: {summary}") + assert_exactly_once(world, model_requests=0) + + +def test_worker_is_idempotent_after_guard_bypassed_stale_upsert_and_hints(): + world, stale = delivered_world_with_stale_copy() + world.runs.upsert_item(body=reverted_pending_copy(stale)) + reset_model_observations(world) + world.hints.signal(USER, RUN_ID) + + hint_outcomes = worker.process_workflow_chat_delivery_hints(world.services) + summary = sweep(world) + outcome = deliver(world) + + require(hint_outcomes == [(RUN_ID, worker.OUTCOME_DELIVERED)], f"unexpected hint outcomes: {hint_outcomes}") + require(summary["processed"] == 0, f"sweep should find no extra work after hint: {summary}") + require(outcome == worker.OUTCOME_NOT_APPLICABLE, f"delivered record should be closed: {outcome}") + assert_exactly_once(world, model_requests=0) + + +def test_worker_does_not_mark_unread_again_when_user_read_after_guard_bypass(): + world, stale = delivered_world_with_stale_copy() + user_reads_chat(world) + world.runs.upsert_item(body=reverted_pending_copy(stale)) + reset_model_observations(world) + world.hints.signal(USER, RUN_ID) + + hint_outcomes = worker.process_workflow_chat_delivery_hints(world.services) + summary = sweep(world) + + require(hint_outcomes == [(RUN_ID, worker.OUTCOME_DELIVERED)], f"unexpected hint outcomes: {hint_outcomes}") + require(summary["processed"] == 0, f"sweep should not post after check-before-compose: {summary}") + assert_exactly_once(world, model_requests=0) + conversation = world.conversation() + require(conversation["has_unread_assistant_response"] is False, "a read chat must stay read after stale revert") + + +def test_guard_restores_missing_delivery_and_invocation_on_chat_trigger(personal_workflows, monkeypatch): + stored = make_run() + container = FakeContainer("runs", [stored]) + install_run_container(monkeypatch, personal_workflows, container) + incoming = { + "id": RUN_ID, + "user_id": USER, + "workflow_id": stored["workflow_id"], + "trigger_source": CHAT_TRIGGER_SOURCE, + "status": "completed", + "progress": {"phase": "done"}, + } + + save_run(personal_workflows, incoming) + + saved = container.get(RUN_ID) + require(saved["chat_delivery"] == stored["chat_delivery"], "stored chat_delivery must be restored") + require(saved["chat_invocation"] == stored["chat_invocation"], "stored chat_invocation must be restored") + require(saved["progress"] == {"phase": "done"}, "incoming runner field must be retained") + + +def test_guard_retries_412_and_keeps_concurrent_delivery(personal_workflows, monkeypatch): + stored = make_run() + container = FakeContainer("runs", [stored]) + install_run_container(monkeypatch, personal_workflows, container) + concurrent_delivery = make_record( + status=STATUS_DELIVERED, + generation=7, + phase=PHASE_MESSAGE_CREATED, + message_id="assistant_workflow_delivery_concurrent", + ) + original_replace = container.replace_item + state = {"failed": False} + + def replace_with_concurrent_delivery(item=None, body=None, etag=None, match_condition=None, **kwargs): + if not state["failed"]: + state["failed"] = True + item_id = item.get("id") if isinstance(item, dict) else item + container.calls.append(("replace_item", item_id, etag)) + current = container.get(RUN_ID) + current["chat_delivery"] = copy.deepcopy(concurrent_delivery) + container.put(current) + raise cosmos_error(412, "concurrent delivery") + return original_replace(item=item, body=body, etag=etag, match_condition=match_condition, **kwargs) + + container.replace_item = replace_with_concurrent_delivery + incoming = copy.deepcopy(stored) + incoming["progress"] = {"runner": "after-conflict"} + incoming["chat_delivery"] = make_record(status="pending") + + save_run(personal_workflows, incoming) + + saved = container.get(RUN_ID) + read_count = sum(1 for call in container.calls if call[0] == "read_item") + replace_count = sum(1 for call in container.calls if call[0] == "replace_item") + require(read_count == 2, f"412 must re-read before retrying, calls={container.calls}") + require(replace_count == 2, f"412 must retry replace, calls={container.calls}") + require(saved["chat_delivery"] == concurrent_delivery, "concurrent stored chat_delivery must win") + require(saved["progress"] == {"runner": "after-conflict"}, "incoming runner fields must survive retry") + + +def test_guard_read_404_upserts_run_record_unchanged(personal_workflows, monkeypatch): + container = FakeContainer("runs") + install_run_container(monkeypatch, personal_workflows, container) + incoming = make_run(record=make_record(status="pending")) + + save_run(personal_workflows, incoming) + + saved = container.get(RUN_ID) + require(saved["chat_delivery"] == incoming["chat_delivery"], "read 404 must upsert incoming delivery unchanged") + require(len(upsert_calls(container)) == 1, f"read 404 must upsert once, calls={container.calls}") + require(replace_calls(container) == [], f"read 404 must not replace, calls={container.calls}") + + +def test_guard_read_failure_logs_and_upserts_current_merge(personal_workflows, monkeypatch): + incoming = make_run(record=make_record(status="pending")) + container = FakeContainer("runs", [incoming]) + container.fail("read_item", cosmos_error(503, "read outage")) + install_run_container(monkeypatch, personal_workflows, container) + incoming["progress"] = {"runner": "read-failed"} + + save_run(personal_workflows, incoming) + + saved = container.get(RUN_ID) + reasons = [entry.get("reason") for entry in personal_workflows.logs] + require("read_failed" in reasons, f"read failure must be logged, logs={personal_workflows.logs}") + require(saved["chat_delivery"] == incoming["chat_delivery"], "read_failed path upserts the incoming merge as-is") + require(saved["progress"] == {"runner": "read-failed"}, "read_failed upsert must include incoming fields") + require(len(upsert_calls(container)) == 1, f"read failure must upsert once, calls={container.calls}") + + +def test_guard_replace_404_upserts_merged_record(personal_workflows, monkeypatch): + stored = make_run() + container = FakeContainer("runs", [stored]) + container.fail("replace_item", cosmos_error(404, "deleted")) + install_run_container(monkeypatch, personal_workflows, container) + incoming = copy.deepcopy(stored) + incoming["chat_delivery"] = make_record(status="pending") + incoming["progress"] = {"runner": "deleted-before-replace"} + + save_run(personal_workflows, incoming) + + saved = container.get(RUN_ID) + expected = merge_stored_run_fields(incoming, stored) + require(saved["chat_delivery"] == expected["chat_delivery"], "replace 404 must upsert merged delivery") + require(saved["progress"] == expected["progress"], "replace 404 must upsert merged runner fields") + require(len(replace_calls(container)) == 1, f"replace 404 must attempt one replace, calls={container.calls}") + require(len(upsert_calls(container)) == 1, f"replace 404 must upsert once, calls={container.calls}") + + +def test_guard_conflicts_exhausted_logs_and_upserts_merged_record(personal_workflows, monkeypatch): + stored = make_run() + container = FakeContainer("runs", [stored]) + container.fail("replace_item", *(cosmos_error(412, "conflict") for _ in range(8))) + install_run_container(monkeypatch, personal_workflows, container) + incoming = copy.deepcopy(stored) + incoming["chat_delivery"] = make_record(status="pending") + incoming["progress"] = {"runner": "after-eight-conflicts"} + + save_run(personal_workflows, incoming) + + saved = container.get(RUN_ID) + expected = merge_stored_run_fields(incoming, stored) + reasons = [entry.get("reason") for entry in personal_workflows.logs] + require("conflicts_exhausted" in reasons, f"exhausted conflicts must be logged, logs={personal_workflows.logs}") + require(len(replace_calls(container)) == 8, f"must attempt exactly 8 replaces, calls={container.calls}") + require(len(upsert_calls(container)) == 1, f"must upsert once after conflicts, calls={container.calls}") + require(saved["chat_delivery"] == expected["chat_delivery"], "conflict exhaustion must upsert merged delivery") + require(saved["progress"] == {"runner": "after-eight-conflicts"}, "incoming runner fields must survive conflicts") + + +def test_guard_non_412_replace_error_propagates_without_upsert(personal_workflows, monkeypatch): + stored = make_run() + container = FakeContainer("runs", [stored]) + container.fail("replace_item", cosmos_error(500, "replace broke")) + install_run_container(monkeypatch, personal_workflows, container) + incoming = copy.deepcopy(stored) + incoming["progress"] = {"runner": "will-not-write"} + + with pytest.raises(CosmosHttpResponseError): + save_run(personal_workflows, incoming) + + require(len(replace_calls(container)) == 1, f"replace should be attempted once, calls={container.calls}") + require(upsert_calls(container) == [], f"replace 500 must not upsert, calls={container.calls}") + saved = container.get(RUN_ID) + require("progress" not in saved, "failed replace must not persist incoming fields") + + +def test_non_chat_run_uses_plain_upsert_without_read(personal_workflows, monkeypatch): + container = FakeContainer("runs") + install_run_container(monkeypatch, personal_workflows, container) + incoming = { + "id": "manual-run-1", + "user_id": USER, + "workflow_id": "workflow-manual", + "trigger_source": "manual", + "status": "completed", + } + + save_run(personal_workflows, incoming) + + read_count = sum(1 for call in container.calls if call[0] == "read_item") + require(read_count == 0, f"non-chat save must not read before upsert, calls={container.calls}") + require(len(upsert_calls(container)) == 1, f"non-chat save must plain upsert once, calls={container.calls}") + + +def test_incoming_cosmos_system_fields_are_not_written_by_merge(personal_workflows, monkeypatch): + stored = make_run() + container = FakeContainer("runs", [stored]) + install_run_container(monkeypatch, personal_workflows, container) + incoming = copy.deepcopy(stored) + incoming.update({ + "_rid": "incoming-rid", + "_self": "incoming-self", + "_etag": "incoming-etag", + "_attachments": "incoming-attachments", + "_ts": 12345, + "progress": {"runner": "system-fields"}, + }) + + save_run(personal_workflows, incoming) + + saved = container.get(RUN_ID) + for field in COSMOS_SYSTEM_FIELDS: + value = saved.get(field) + require(value != incoming[field], f"incoming {field} must not be written back as-is") + require(saved["progress"] == {"runner": "system-fields"}, "ordinary incoming fields must still be written") + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From 1542f83170ad8184e9c6d6b23f0f2e65f57bf710 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 03:41:54 -0400 Subject: [PATCH 12/36] Document workflow result delivery to chat and its 6b-2 contract; add release notes (6b-1) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/admin/orchestration.md | 8 + docs/admin/workflow.md | 8 + .../CHAT_ORCHESTRATION_WORKFLOW_RUNS.md | 9 + .../CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md | 3 + .../features/CHAT_WORKFLOW_RESULT_DELIVERY.md | 846 ++++++++++++++++++ docs/explanation/release_notes.md | 11 + docs/guides/trigger-a-workflow.md | 6 + 7 files changed, 891 insertions(+) create mode 100644 docs/explanation/features/CHAT_WORKFLOW_RESULT_DELIVERY.md diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index eee7bdb13..430155d23 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -281,6 +281,13 @@ not claim that research verified the requested details. | Run Workflows From Chat | Lets a plan start one of the user's saved personal workflows when the user asks for it, such as "run my weekly digest now". A plan that starts a workflow always waits for the user to run it, even when the approval mode would run it automatically or after a countdown, and approving it starts each named workflow once. Runs are offered only in the user's own private conversations, only to users who may already create personal workflows, and only for workflows with durable execution on. It is independent of Propose Workflows From Chat, so either can be on without the other. | Off | `enable_chat_orchestration_workflow_runs`; requires Chat Orchestration and personal workflows (`allow_user_workflows`), and the **Run workflows** capability when the Capabilities list is narrowed. Since **0.261.212** | | Capabilities | Restricts which capabilities a plan may use. An empty selection means every capability the other settings already permit. | All | `chat_orchestration_enabled_capabilities` | +When **Run Workflows From Chat** starts a durable personal workflow, the plan +still ends right away. If **Use Workflow Results In Chat** is also on in +Workflow settings, the server records enough context to post the run's eventual +outcome back to the same private chat later, mark it unread and send one bell +notification. If the chat can no longer take the result, the server posts +nothing and sends one workflow notification instead. + ### Actions and agents Use **Use an action** for a question that needs a particular integration, such as a ticket @@ -548,6 +555,7 @@ then report a rejected proposal. | A workflow step says saved workflows were temporarily unavailable | Workflow storage could not be read or written when the step ran, so the workflow may not have started. It is not retried automatically. | Once storage is reachable, the user can select **Retry from failed step**. A run the plan already started is linked rather than started again. | | Retry requires confirmation | An agent/action may have performed external effects before it failed, or a step may already have started a saved workflow. | Review those effects before confirming. Retry reexecutes that failed step, not its internal tool-call checkpoint. A workflow the plan already started is linked again, never started twice. | | A saved run cannot be resumed | Checkpoints are absent or invalid, relevant context/access changed, or a newer/live attempt exists. | Follow the recovery explanation. Open the current attempt or create a new plan as appropriate; do not infer results from old summaries. | +| A chat-started workflow finishes but no result appears in the chat | The workflow settings gate for result post-back is off, the chat was deleted or shared after the run started, access to personal workflows was lost, the workflow was deleted, or the delivery worker deferred or closed the generation after retryable storage/notification failures. | Check **Use Workflow Results In Chat** under Workflow settings, keep the request in the user's private chat, and open the run from Workflows or the workflow notification. The plan itself is already complete and does not wait for delivery. | ## Related diff --git a/docs/admin/workflow.md b/docs/admin/workflow.md index 57861a78c..c6594ed92 100644 --- a/docs/admin/workflow.md +++ b/docs/admin/workflow.md @@ -65,6 +65,13 @@ visible to its members. Turning one on does not turn on the other. | Workflow Repeat Iteration Limit | Bounds one automatic Repeat until batch, including its first round. Authors must choose an explicit per-block maximum; a new run above this ceiling is rejected rather than shortened. | 25 | `workflow_max_repeat_iterations`; supported range 1-1,000; new runs only; active runs and manual continuation retain the admitted policy | | Workflow Minimum Schedule Interval (seconds) | Sets the shortest fixed interval a personal or group workflow may be saved with, so no one can schedule a workflow to run every few seconds. It's checked only when an interval schedule is created or changed; workflows already saved on a shorter interval keep running. Calendar schedules run at most once a day and are never checked. The default, 1 second, allows every interval, as before. | 1 | `workflow_min_schedule_interval_seconds`; supported range 1-86,400; new or changed interval schedules only | +When **Use Workflow Results In Chat** is on, the same gate also controls whether +a durable personal workflow started from chat can post its terminal outcome back +into that private chat later. The server checks the gate when the chat starts +the run and again before it reads or posts result text, so turning it off leaves +future chat-started runs without a post-back and makes already-started runs post +a fixed status note instead of result text. + The action and task limits apply to personal and group runs alike, so they stay in effect whichever capability is enabled. @@ -274,6 +281,7 @@ See [Workflow publication completion](../explanation/features/WORKFLOW_PUBLICATI | The AI workflow assistant is temporarily unavailable for everyone | The model deployment is throttled or unreachable, or the settings container that holds the rate-limit count can't be reached. The assistant won't run without its limit. | Check the GPT deployment's health and throttling, and Cosmos DB availability. Each request logs `[WorkflowAssist] Assist request finished` with its `status` and error `code`, and no content. | | Users don't see **Ask in chat** or **Ask about this** in V2 | Use Workflow Results In Chat or Enable Personal Workflows is off, or Require WorkflowUser App Role is on and the user doesn't hold the role. Both buttons also need a completed run of a personal workflow, and **Ask in chat** isn't offered on structured runs. | Check both settings, then the `WorkflowUser` role assignment. | | A chat answer reads "This answer is unavailable because access to the workflow result it used could not be confirmed." | The run was deleted, its result changed, its owner can no longer read a source its tasks used, the chat was shared or converted to a collaboration, or storage couldn't be read during the check. | Expected. An answer is shown only while its result still checks out for the chat's owner in a private chat; after a storage failure it shows again once a check succeeds. The original chat's stored messages are unchanged. The collaboration's copies are stored with these answers withheld, and the chat's saved summary is cleared on both chats. Classic chat's message list doesn't run this check, so the owner still sees the answer there. Each withheld answer logs `[WorkflowResults] A chat answer's workflow result was withheld on read.` with a `check` of `conversation` or `result`, a `code` or `error_type`, and no content. | +| Results of a workflow started from chat never appear in the chat | Use Workflow Results In Chat was off when the chat started the run, the requester no longer passes the personal workflow gate, the chat was deleted or shared, the workflow was deleted, the runtime expired, or delivery closed after repeated storage/notification failures. | Check **Use Workflow Results In Chat**, **Enable Personal Workflows**, the `WorkflowUser` role, the chat's privacy, and the run in workflow history. Deleted workflows and missing runtime records close silently; other undeliverable runs create one workflow notification with a run link. | ## Related diff --git a/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md b/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md index fa65b8a20..05381725c 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION_WORKFLOW_RUNS.md @@ -25,6 +25,15 @@ starts the run and links to it. It never waits for the run and never reads its results. A plan that starts a workflow always waits for the user to run it, whatever approval mode they chose. +Since version **0.261.218**, when **Use Workflow Results In Chat** is also on, +the server can post a chat-started run's outcome back into the chat that asked +after the run finishes. The started-run note changes from pointing only to run +history to "I'll post the results here when the run finishes. You can also +follow it in the workflow's run history in Workflows." For multiple runs it says +"I'll post each run's results here when it finishes. You can also follow them +in each workflow's run history in Workflows." See +[Workflow result delivery to chat](CHAT_WORKFLOW_RESULT_DELIVERY.md). + What this version adds: - **The `workflow_run` capability.** One plan step per workflow, naming it by a diff --git a/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md b/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md index 0a542c82b..c4c982add 100644 --- a/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md +++ b/docs/explanation/features/CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md @@ -724,3 +724,6 @@ the controls are listed in - Re-checking workflow result contexts when a generated file is published. - Phase 6b's delivery, run card, recurring-workflow card and chat-list indicator, and `v2WorkflowRunPath`. +- Phase 6b-1 now has its server-side delivery contract in + [Workflow result delivery to chat](CHAT_WORKFLOW_RESULT_DELIVERY.md); the V2 + run card and chat-list indicator still follow in later work. diff --git a/docs/explanation/features/CHAT_WORKFLOW_RESULT_DELIVERY.md b/docs/explanation/features/CHAT_WORKFLOW_RESULT_DELIVERY.md new file mode 100644 index 000000000..73cdb4478 --- /dev/null +++ b/docs/explanation/features/CHAT_WORKFLOW_RESULT_DELIVERY.md @@ -0,0 +1,846 @@ +# Workflow result delivery to chat + +Implemented in version: **0.261.218**. + +Application version tracking: `application\single_app\config.py`. + +Related issue: #1546 (part 6b-1), part of #1543. Builds on +[Chat orchestration workflow runs](CHAT_ORCHESTRATION_WORKFLOW_RUNS.md) +(#1594), [Workflow results in chat](CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md) +(#1546 part 6a), and the V2 workflow run link described in +[Document provenance](DOCUMENT_PROVENANCE.md) (#1562). The +[Chat orchestration workflows roadmap](CHAT_ORCHESTRATION_WORKFLOWS_ROADMAP.md) +describes all the phases; this is the server-side part of Phase 6b. + +## Overview and dependencies + +A chat orchestration plan can start one of the requester's saved personal +workflows and then finish immediately. Before this version, the user had to open +Workflows later to see what the run produced. With **Use Workflow Results In +Chat** on, the server posts that run's terminal outcome back into the same +private chat that asked for it, even if the run finishes hours later. + +The delivery is server-side. It ships no V2 run-card UI in this pull request. +6b-2 polls the status route documented below and decides how to render cards, +Retry and Open run. + +What this version adds: + +- a `chat_delivery` run record seeded into the run's first durable write when a + chat plan starts a personal durable workflow and the role-aware **Use Workflow + Results In Chat** gate passes. +- runtime projection reconciliation that moves that record from pending to ready + at the terminal durable runtime generation, reopens it only for a later resume + generation, expires old in-flight records, and signals a worker without + blocking the run. +- an idempotent delivery worker that re-checks chat ownership, privacy, + workflow access and the gate; defers while a chat may be mid-stream; composes + a bounded result summary; posts one deterministic message per generation; + marks the chat unread; and sends one bell notification. +- fixed application notes for failures, cancellations, skipped runs, saved + analyses, content-blocked results and results that cannot be shown in chat. +- undeliverable notices when the chat cannot accept the post any more, such as + a deleted chat, a shared chat or lost access. Deleted workflows and missing + runtime controls close silently. +- classic Retry/Edit refusal for any message a workflow run posted. +- an owner-only status route for 6b-2: + `GET /api/v2/orchestration/workflow-runs/status[?conversation_id=...]`. + +Dependencies: + +- Personal workflows (`allow_user_workflows`). When + `require_member_of_workflow_user` is on, the requester also needs the + `WorkflowUser` app role. +- Chat orchestration run steps (`enable_chat_orchestration_workflow_runs`) from + Phase 5. Delivery never starts a workflow by itself. +- Durable execution. Chat-started workflow runs eligible for delivery are the + durable runs started by Phase 5. +- Workflow results in chat (`enable_chat_workflow_results`), off by default. + The role-aware gate `is_chat_workflow_results_enabled_for_user` is checked at + start and again at delivery. +- The background task host and the durable workflow scheduler. The worker uses a + hint queue for prompt delivery and a locked sweep as the backstop. + +## Technical specifications + +### Architecture and flow + +```mermaid +sequenceDiagram + participant Chat as Chat orchestration + participant Runtime as Durable runtime + participant Runs as personal_workflow_runs + participant Worker as Delivery worker + participant Messages as Chat messages + participant Bell as Notifications + + Chat->>Runtime: queue durable run with chat_invocation and optional chat_delivery + Runtime->>Runs: create run document and control + Runtime->>Runs: project terminal control.version onto run + Runtime->>Runs: reconcile chat_delivery to ready(kind, generation) + Runtime-->>Worker: signal hint(user_id, run_id) + Worker->>Runs: ETag claim ready generation + Worker->>Worker: re-check chat, workflow, gates and stream activity + Worker->>Runtime: read bounded workflow result excerpt when kind=result + Worker->>Worker: compose or choose fixed application note + Worker->>Chat: mark conversation unread for deterministic message id + Worker->>Messages: create assistant_workflow_delivery_* message if absent + Worker->>Bell: create one idempotent notice + Worker->>Runs: close delivered, undeliverable or expired +``` + +The plan does not wait for the workflow. Its started-run note explains where the +result will appear. When every run that answer started carries a delivery seed, +the note uses these fixed strings from the contract module: + +- `I'll post the results here when the run finishes. You can also follow it in the workflow's run history in Workflows.` +- `I'll post each run's results here when it finishes. You can also follow them in each workflow's run history in Workflows.` + +Linked or rebuilt runs keep the older Phase 5 note text, because only a fresh +start that recorded this chat's delivery sets the sidecar `chat_delivery: true`. + +### The `chat_delivery` record + +The run document holds a `chat_delivery` object. It contains only IDs, statuses, +model identity fields, roles, timestamps, lease data and closed reason codes. It +never stores result text, prompt text, model output or exception text. + +| Field | Meaning | +| --- | --- | +| `version` | Contract version, currently `1`. | +| `status` | `pending`, `ready`, `delivering`, `delivered`, `undeliverable` or `expired`. | +| `generation` | Durable runtime control `version` for the terminal projection being delivered, or `null` before one is ready. | +| `expires_at` | Delivery deadline. Runtime `deadline_at + 24h` when available, otherwise seed time plus the admitted deadline, capped at 86,400 seconds, plus 24 hours. | +| `time_zone` | Request time zone, or planning time zone, or `UTC`. | +| `model_selection` | Bounded request model identity: `model_deployment`, `model_id`, `model_endpoint_id`, `model_provider`, `reasoning_effort` and up to 10 group ids. | +| `requester_roles` | Up to 20 role names, each up to 64 characters, captured at start for headless gate checks. | +| `lease_id`, `lease_expires_at` | The ETag claim fence. Leases last 300 seconds. | +| `attempts`, `next_attempt_at`, `first_deferred_at` | Retry and chat-settling state. Backoff is 30s, 1m, 2m, 4m, 8m, 15m, 15m, 15m; at 8 attempts the record closes. | +| `phase` | `claimed`, `publishing`, `unread_marked`, `message_created`, `notified` or `null`. | +| `message_id`, `planned_at` | The deterministic message id and the planned unread/message time. | +| `kind` | `result`, `failed`, `cancelled`, `status`, `content_blocked`, `analysis`, `expired` or `skipped`. | +| `notice_kind` | `chat_response`, `undeliverable`, `expired`, `none` or `null`. | +| `outcome_reason` | Closed code such as `chat_unavailable`, `access_lost`, `workflow_deleted`, `runtime_missing`, `delivery_failed`, `expired_before_delivery`, `content_blocked`, `results_off`, `result_unavailable` or `deadline_exceeded`. | +| `run_status` | Runtime state observed for the generation. | +| `delivered_at`, `created_at`, `updated_at` | UTC timestamps written in the delivery format. | +| `history` | Up to five summaries of older generations, with codes and ids only. | + +The origin fields stay in `chat_invocation`: `conversation_id`, +`user_message_id`, `orchestration_run_id`, `attempt_root_run_id`, `step_id`, +`requested_by` and `requested_at`. + +### Generation and reopen rule + +A delivery generation is the durable runtime control `version` at the terminal +projection. The message id, notification idempotency key and status row all use +that generation. + +A `delivered` or `undeliverable` record reopens only when the live control's +`version` is greater than the record's `generation` and the control is not +`deleted`. Durable resume is the normal source of that version bump. The resume +route keeps the same `run_id` and expects the client body +`{expected_version, request_id}`. + +Tombstone deletion is the only non-resume write that bumps a terminal control's +version. It sets `deleted: true`, so it does not reopen delivery. A deleted +control closes a pending or ready record silently as `workflow_deleted`. A +missing control closes within one sweep as `runtime_missing`, also silently. +Expired records never reopen. An already delivered record stays delivered when a +control is deleted later. + +### Guarded save and exactly-once delivery + +There are two layers that keep delivery exactly once: + +1. `save_personal_workflow_run` uses a guarded, merge-preserving ETag path for + runs that carry `chat_delivery`, and for the double-fault case where a + chat-started run lacks `chat_invocation`. The stored `chat_delivery` always + wins over an incoming stale copy. The stored `chat_invocation` and stored + cancellation request are preserved where needed. +2. The worker's side effects are idempotent. It claims with ETag and a lease, + checks for the deterministic message before composing, uses one message id + per `(run_id, conversation_id, generation)`, treats message-create 409 as + success, marks unread under an ETag guard, and sends notifications with + deterministic keys. + +The deterministic message id is: + +```text +assistant_workflow_delivery_{sha256(canonical_json([run_id, conversation_id, generation]))[:40]} +``` + +The thread id is `uuid5(NAMESPACE_URL, message_id)`. + +### Worker phases + +For each due generation the worker: + +1. **Claims and reconciles.** It point-reads the run, reads the runtime control + with deleted controls allowed, reconciles the delivery record, and ETag + replaces the record to `delivering` with a fresh lease and `phase: claimed`. +2. **Skips ahead if the message exists.** After it reads settings and the + workflow, and before any other check, it point-reads the deterministic + message id unless the record already says the message was created. If the + message exists, because a crash lost the phase write or a stale save reverted + the record, it records `message_created` and goes straight to the notice: no + compose, no unread mark and no second message. A failed read retries later. +3. **Re-checks.** It requires the original conversation to exist, belong to the + requester, not be deleted or `orchestration_deleted`, and still be private. + It requires the workflow to exist and not be `deleting`, and it re-checks + personal workflow access from live settings and the stored requester roles. +4. **Defers while the chat may be busy.** It checks shared stream activity, + fresh user messages and newest-message read failures. Soft activity signals + stop delaying after 15 minutes; a failed newest-message read still defers and + consumes an attempt. +5. **Builds content.** Result generations read bounded excerpts and may compose; + failed, cancelled, skipped, status, analysis and content-blocked paths use + fixed application text. +6. **Marks unread before creating the message.** The helper re-reads the + conversation, re-checks owner/privacy/delete state, raises `last_updated` to + the planned message time and writes unread fields with ETag. Retries keep + `skip_if_read_since`, so a user who already read the chat is not re-marked. +7. **Creates the message.** It freshly reads the newest message, places the + delivery after it by timestamp, carries the previous thread id, and creates + the assistant message. A 409 counts as success. +8. **Notifies.** Delivered messages use the existing chat-response notification + helper. Undeliverable and expired runs use the new + `workflow_chat_delivery` notification type. Keys make each notice exactly + once. +9. **Finishes.** It closes the record as `delivered`, `undeliverable` or + `expired`, and reconciles with the live control in that final write so a + resume that landed mid-delivery can reopen the next generation. + +If attempts are exhausted before a claim can be completed, the worker closes the +record silently as `delivery_failed`. If a message was already created, it +finishes as delivered instead of sending an undeliverable notice. + +### Compose, fencing and token logging + +For `kind: result`, the worker reads the result with `include_excerpts=True` and +an initial 48 KB excerpt budget. If the prompt does not fit a model's window, it +steps down through the same 6a budget ladder: 24 KB, 12 KB and 6 KB. If no +budget fits, it posts the fixed status note instead of result text. + +The question is the original user message if it still exists and was not +retracted. Otherwise the worker uses the fixed fallback: + +```text +Summarize what this workflow run produced. +``` + +No chat history is sent. The dropped plan-step-goal fallback is not in the +implementation. + +The model chain is: + +1. the request's stored model selection, re-authorized for the requester with a + headless execution identity for the conversation; +2. the default model, skipped when it resolves to the same binding; +3. a trimmed fallback if no model answers: the fixed sentence + `I couldn't summarize the result, so here's the start of it:`, then the first + 2,000 characters of the excerpt in a code block whose fence is longer than + any run of backticks in the data. + +The worker calls 6a's `build_workflow_result_messages` with a fresh fence nonce +on every attempt. The result is fenced as untrusted data. It then calls the +model with `temperature=0.3` and `max_tokens=1500`, clips the reply at 12,000 +characters, and appends 6a's disclosure line. + +The final composed or trimmed text is content-checked as `chat_output`. A +blocked result posts the fixed `content_blocked` note instead, records a +best-effort content incident without excerpt text, and does not retract the +message. Fixed notes are application-written and are not model-checked. + +When a model returns usage, the worker logs token usage like 6a Follow up: +`token_type='chat'`, `workspace_type='personal'`, `additional_context` of +`{'workflow_result_delivery': True}`, and idempotency key +`workflow_result_delivery:{message_id}:{attempt}`. It does not log a chat +activity user message. Token logging failures never fail delivery. + +### Labels and application-owned texts + +The label line is: + +```text +Results from `{workflow name}` ยท you asked on {formatted requested_at} +``` + +Workflow names are cleaned to one line, capped at 80 characters and rendered as +inline code with backticks replaced by apostrophes. If no name remains, the +label uses `` `Workflow` ``. The timestamp uses the same formatter as 6a and the +record's time zone. + +Fixed note bodies from `delivery_note_text` are: + +| Kind | Exact text pattern | +| --- | --- | +| `failed` | `` `{name}` failed: {closed reason text}. `` | +| `cancelled` | `` `{name}` was cancelled. `` | +| `status` | `` `{name}` finished. Open the run to see its results. `` | +| `content_blocked` | `` `{name}` finished, but its results can't be shown here. Open the run to see them. `` | +| `analysis` | `` `{name}` finished with a saved analysis. Ask a follow-up question about it here. `` | +| `skipped` | `` `{name}` didn't run: no new or changed files were detected. `` | + +The failed reason comes only from a closed map: `failed`, `invalid`, +`incomplete`, `skipped`, `deadline_exceeded`, `execution_budget_exceeded`, +`repeat_iteration_limit`, `m365_authorization` and `authorization`. Unknown +failure codes use `the run stopped before it finished`. + +Notice titles and messages are fixed: + +| Case | Type | Title or preview | Message | Key | +| --- | --- | --- | --- | --- | +| Delivered | `chat_response_complete` through `create_chat_response_notification` | `Results from "X" are in your chat` as a non-empty preview | Existing helper message, opening the chat | `workflow-chat-delivery:{run_id}:{generation}` | +| Undeliverable completed/partial/status/content-blocked/analysis | `workflow_chat_delivery` | `Results from "X" are ready` | `The chat that started this run can't show it anymore. Open the run in Workflows to see the details.` | `workflow-chat-delivery-notice:{run_id}:{generation}` | +| Undeliverable failed | `workflow_chat_delivery` | `"X" didn't finish` | Same undeliverable message | Same undeliverable key | +| Undeliverable cancelled | `workflow_chat_delivery` | `"X" was cancelled` | Same undeliverable message | Same undeliverable key | +| Undeliverable skipped | `workflow_chat_delivery` | `"X" didn't run` | Same undeliverable message | Same undeliverable key | +| Expired | `workflow_chat_delivery` | `"X" didn't finish in time to post to chat` | `The run didn't finish in time to post to the chat. Open it in Workflows to see where it stands.` | `workflow-chat-delivery-notice:{run_id}:expired` | + +Undeliverable and expired notices link to +`/workflow-activity?workflowId=...&runId=...&scope=personal` and carry metadata +`{workflow_id, run_id, workflow_scope: 'personal', delivery_status}`. Until 6b-2 +adds a V2 case, the V2 bell labels the new type generically as `Notification`. + +### Message shape and placement + +The delivered message document is an assistant message shaped like a normal chat +answer: + +```json +{ + "id": "assistant_workflow_delivery_...", + "conversation_id": "...", + "role": "assistant", + "content": "Results from `Weekly digest` ...", + "timestamp": "2026-01-05T02:37:11.000001", + "model_deployment_name": "gpt-4o", + "augmented": false, + "hybrid_citations": [], + "hybridsearch_query": null, + "agent_citations": [], + "web_search_citations": [], + "user_message": null, + "metadata": { + "workflow_delivery": {"version": 1}, + "token_usage": {}, + "user_info": {"user_id": "..."}, + "thread_info": {"thread_id": "...", "previous_thread_id": "...", "active_thread": true, "thread_attempt": 1} + } +} +``` + +The worker reads the newest message with `ORDER BY c.timestamp DESC`; a failed +read defers rather than treating the chat as empty. The delivery timestamp is +later than the newest timestamp by one microsecond when necessary, so loaders +sort it last. The message does not carry `metadata.orchestration`; it is not an +orchestration answer. + +### Retry and Edit refusal + +`is_workflow_delivery_message(message)` is true when the id starts with +`assistant_workflow_delivery_` or `metadata.workflow_delivery` is present. +Classic Retry and Edit refuse those messages before replay work begins: + +| Operation | Response | +| --- | --- | +| Retry | HTTP 400 `{"error": "A workflow run posted this message, so it can't be retried here. To run the workflow again, open the run in Workflows.", "code": "workflow_delivery_retry_unsupported"}` | +| Edit | HTTP 400 `{"error": "A workflow run posted this message, so it can't be edited. Ask a new question instead.", "code": "workflow_delivery_edit_unsupported"}` | + +V2 orchestration retry/edit routes take an orchestration run id, not a message +id, so a delivery message id is not a delivery trigger there and returns the +existing 404 path. + +### Masking and lineage + +A delivered result answer and an analysis note carry the same lineage keys as a +6a Follow up answer: + +- `metadata.workflow_result`: the reader's public descriptor snapshot. +- `metadata.workflow_result_contexts`: an array containing the context built + from that descriptor. + +Those keys make later chat turns inherit the result lineage and make shared +sanitizers withhold the delivered content when the run is gone, the result +changed, source access is lost, storage cannot confirm access, or the chat is no +longer private. Status, failed, cancelled, skipped and content-blocked notes +carry no result text and no 6a lineage keys. + +Every delivered message carries `metadata.workflow_delivery`, described in the +message metadata contract below. The id prefix is a second marker and survives +content-review retraction. + +### Expiry, deferral, sweep and lease + +A record can become `expired` when the runtime pauses with `deadline_exceeded`, +or when `now > expires_at` and the run is still non-terminal. The worker posts +nothing and sends one expired notice. If a run finished but stayed undelivered +past `expires_at`, it becomes `undeliverable` with +`expired_before_delivery` and a ready notice. Expired records never reopen. + +The worker delays delivery while the chat may still be receiving the +orchestration answer or a user reply: shared stream activity is active and fresh +within 90 seconds, the newest message is a user message less than 10 minutes +old, or the newest-message read fails. The first uncounted deferral records +`first_deferred_at` and retries after 30 seconds. After 15 minutes, soft +activity signals are ignored and delivery continues; read failures still defer +and count as attempts. + +`signal_workflow_chat_delivery(user_id, run_id)` puts a bounded in-process hint +onto a 256-entry queue and wakes the delivery loop. Hints are point reads plus +an ETag claim, so they do not need the distributed sweep lock. + +`run_workflow_chat_delivery_loop(app)` runs inside the Flask app context when an +app is provided. On each wake it processes hints and, every 30 seconds, attempts +to acquire the distributed lock `workflow_chat_delivery_sweep` for 120 seconds. +With the lock it runs one cross-partition sweep over due `chat_delivery` +records. The sweep uses `TOP 25`, processes at most 8 runs or 90 seconds, and +uses a 120-second `_ts` grace so normal runtime projection can win. A claim +lease lasts 300 seconds. A slow model call may be duplicated after lease expiry, +but only the worker that still owns the lease can persist the message phase. + +### Terminal-write coverage + +| # | Terminal or reopen write | Coverage | +| --- | --- | --- | +| 1 | Runner cancellation saves and cancellation merge writes | Guarded save preserves delivery; runtime projection reconciles and signals; sweep catches pending terminal rows after grace. | +| 2 | Runner failure saves (`failed`) | Guarded save, projection trigger and sweep backstop. | +| 3 | Runner success and partial success saves (`completed`, `completed_partial`) | Guarded save, projection to `ready` with `kind: result`, signal and sweep. | +| 4 | Runner final exception save (`failed`) | Guarded save, projection and sweep. | +| 5 | Non-terminal runner progress saves | Guarded save preserves `chat_delivery` and `chat_invocation`, including the double-fault path. | +| 6 | Durable projection `_project_runtime_run` | Keeps stored delivery, reconciles under ETag and signals when a generation becomes ready. | +| 7 | Runtime deadline expiry on the control | Next continue projects `paused` + `deadline_exceeded`; sweep also catches `expires_at`. | +| 8 | Durable cancel and reject decisions | Projection maps the terminal state to cancelled or failed note kinds and signals. | +| 9 | Scheduler continuations | Covered through durable projection. | +| 10 | Personal cancel routes | Save through the guarded path; sweep catches pending terminal rows. | +| 11 | `runtime/resume` | Control version increases; projection reopens delivered or undeliverable records when `version > generation` and not deleted. | +| 12 | `runtime/decision` approve/reject | Approve returns to running; reject terminal states are handled through projection. | +| 13 | `resume-failed` | Not a trigger. It returns 409 for durable runs, and any non-durable replacement would have no `chat_invocation`. | +| 14 | Workflow deletion | Deleted workflow with a run present closes silently as `workflow_deleted`; deleted runs have nothing to deliver. | +| 15 | MCP and group workflow saves | Out of scope because they do not carry `chat_delivery`; group runs are for a later phase. | +| 16 | Signal failure | Projection never fails because the signal is non-raising; the sweep catches up later. | + +## Status route contract for 6b-2 + +### Route, decorators and query parameters + +Path: + +```text +GET /api/v2/orchestration/workflow-runs/status[?conversation_id=] +``` + +Decorators, in order: + +1. `@bp.route(..., methods=["GET"])` +2. `@swagger_route(security=get_auth_security())` +3. `@login_required` +4. `@user_required` +5. `@enabled_required('allow_user_workflows')` +6. `@workflow_user_required` + +The optional `conversation_id` may appear once. It is trimmed and must match +`[A-Za-z0-9_-]{1,128}`. Missing means global mode. Empty, repeated, too long or +malformed values return HTTP 400: + +```json +{"error": "The conversation ID is not valid.", "code": "invalid_conversation_id"} +``` + +An invalid server-side user id or storage/settings failure returns HTTP 503: + +```json +{"error": "Workflow run status isn't available right now. Try again later.", "code": "workflow_run_status_unavailable"} +``` + +### Query modes and limits + +All reads are owner-only by construction: the query uses +`partition_key=`, filters `c.user_id = @user_id`, filters +`c.trigger_source = 'chat_orchestration'`, and requires +`IS_DEFINED(c.chat_invocation)`. It projects the needed fields only; it does not +use `SELECT *`. + +| Mode | Query limit | Returned rows | Filter | Order | +| --- | --- | --- | --- | --- | +| With `conversation_id` | `TOP 21` | Up to 20 | That chat's started runs | `ORDER BY c.chat_invocation.requested_at DESC` | +| Without `conversation_id` | `TOP 51` | Up to 50 | Non-terminal runs, open delivery records, or deliveries from the last 10 minutes | Same order | + +`truncated` is true when the query returned more than the public limit. The +runs container uses the default indexing policy; every chat invocation writer +sets `requested_at`. + +For in-flight rows, the route may do at most 10 live runtime reads per request. +It may do at most 20 workflow document reads per request, cached per distinct +workflow and counted separately. When a live read fails or the cap is reached, +the row falls back to stored projection values with `live: false` and fixed +closed codes. + +`available` reflects the Phase 5 `workflow_run_settings_gate` at request time. +Rows are still returned when `available` is `false`, so an in-flight card does +not vanish when an administrator changes the setting mid-run. + +### Response example + +```json +{ + "available": true, + "runs": [ + { + "workflow_id": "wf_123", + "workflow_scope": "personal", + "run_id": "run_123", + "workflow_name": "Weekly digest", + "conversation_id": "conv_123", + "orchestration_run_id": "orch_123", + "step_id": "step_2", + "requested_at": "2026-01-05T02:30:00Z", + "status": "running", + "phase": "running", + "runtime_version": 3, + "step_index": 2, + "step_count": 5, + "step_label": null, + "started_at": "2026-01-05T02:30:15Z", + "completed_at": null, + "elapsed_seconds": 416, + "waiting": null, + "delivery": { + "status": "pending", + "generation": 3, + "message_id": null, + "delivered_at": null, + "reason": null + }, + "error": null, + "error_code": null, + "retry_blocked": null, + "actions": { + "cancel": true, + "retry": false, + "approve": false, + "open_run": true + }, + "live": true + } + ], + "checked_at": "2026-01-05T02:37:11Z", + "truncated": false +} +``` + +### Field rules + +| Field | Rule | +| --- | --- | +| `workflow_id`, `run_id`, `conversation_id`, `orchestration_run_id`, `step_id` | Bounded identifiers from the projected run row. Rows missing required ids are omitted. | +| `workflow_scope` | Always `personal`. | +| `workflow_name` | Cleaned to one line and capped at 80 characters, or `Workflow`. | +| `requested_at`, `started_at`, `completed_at`, `checked_at` | Seconds-precision UTC strings when known. `completed_at` is null for non-terminal rows. | +| `runtime_version` | Live runtime version when a live read succeeded, otherwise the run's `runtime_version`, otherwise the stored projection version. This is the `expected_version` for resume. | +| `step_index`, `step_count` | From live progress, stored runtime progress, or stored run progress. | +| `step_label` | Always `null` in this implementation. | +| `elapsed_seconds` | From `started_at` to `completed_at` for terminal rows, or to `checked_at` for active rows. Null when `started_at` is missing. | +| `live` | True only when a live runtime read succeeded for an in-flight row. | + +Status mapping: + +| Runtime state | `status` | `phase` | +| --- | --- | --- | +| `queued` | `queued` | `running` | +| `running`, `cancelling`, `waiting_recovery` and unknown states | `running` | `running` | +| Other waiting states | `waiting` | `needs_you` | +| `completed` | `completed` | `finished` | +| `completed_partial` | `completed_partial` | `finished` | +| `failed`, `invalid`, `incomplete`, `skipped` | `failed` | `failed` | +| `cancelled` or `canceled` | `cancelled` | `cancelled` | +| Non-terminal row with delivery `expired`, or paused with `deadline_exceeded` | `expired` | `failed` | + +Waiting mapping: + +| Runtime state | `waiting.reason` | `waiting.action` | `waiting.gate_id` | +| --- | --- | --- | --- | +| `waiting_approval` | `approval` | `approve` | Runtime gate id or null | +| `awaiting_sign_in` | `microsoft_365_reconnect` | `reconnect` | Runtime gate id or null | +| Other Microsoft 365 waiting states | `microsoft_365_approval` | `open_run` | Runtime gate id or null | +| `waiting_output` | `output_review` | `open_run` | Runtime gate id or null | +| `waiting_recovery` | `recovery` | `open_run` | Runtime gate id or null | +| `paused` with `deadline_exceeded` | `deadline_exceeded` | `open_run` | Runtime gate id or null | +| Other `paused` | `paused` | `open_run` | Runtime gate id or null | + +Delivery mapping: + +| Stored delivery state | Row `delivery.status` | Other fields | +| --- | --- | --- | +| Missing or invalid | `not_applicable` | Null generation, message, delivered time and reason. | +| `ready` | `pending` | The generation is included when known. | +| `pending`, `delivering`, `undeliverable`, `expired` | Same value | `reason` is returned only when it is one of the closed delivery reason codes. | +| `delivered` | `delivered` | `message_id` is returned only when it starts with `assistant_workflow_delivery_`; `delivered_at` is seconds precision. | + +`error` and `error_code` are fixed and never come from the run's raw error text +or exception text. `expired` returns `It reached its time limit.` with +`deadline_exceeded`. Failed rows use the same closed failure map as notes. + +`retry_blocked` is null or one of these codes, checked in this order: +`retry_unavailable`, `workflow_deleted`, `not_resumable`, +`deadline_exceeded`, `workflow_definition_changed` or +`workflow_already_running`. The implementation also returns +`retry_unavailable` for failed workflow reads, workflow-read cap exhaustion, +missing definition revisions, or revision-computation failures. `skipped` and +`cancelled` are never retryable. + +`actions` mapping: + +| Action | True when | +| --- | --- | +| `cancel` | The row is non-terminal, not expired and not `cancelling`. | +| `retry` | State is `failed`, `invalid` or `incomplete`, and `retry_blocked` is null. | +| `approve` | Waiting action is `approve`, a `gate_id` is present, and the row is not expired. | +| `open_run` | Always true. | + +6b-2 must treat the row as a snapshot. Retry buttons should still handle a 409 +from resume, including `workflow_definition_changed`, +`workflow_already_running`, `workflow_deleting`, `stale_version`, +`invalid_state`, `deadline_exceeded` and `request_conflict`, plus 400, 403, 404 +and 503 responses. + +### Personal run routes used by 6b-2 + +6b-2 calls existing personal durable run routes: + +- `GET /api/user/workflows//runs//runtime` +- `POST /api/user/workflows//runs//runtime/decision` +- `POST /api/user/workflows//runs//runtime/resume` with + body `{expected_version, request_id}`, where `request_id` is a UUID generated + by the client for that resume request. +- `POST /api/user/workflows//runs//cancel` + +The older `POST /api/user/workflows//runs//resume-failed` +route is not used for delivery. It returns 409 for durable runs and would create +a new run id for non-durable runs. + +## Message metadata contract + +### `metadata.workflow_delivery` + +Every posted delivery message, including fixed notes, carries: + +```json +{ + "version": 1, + "kind": "result", + "workflow_id": "wf_123", + "workflow_scope": "personal", + "run_id": "run_123", + "generation": 3, + "run_status": "completed", + "orchestration_run_id": "orch_123", + "step_id": "step_2", + "requested_at": "2026-01-05T02:30:00Z" +} +``` + +This metadata is a snapshot. 6b-2 uses it to identify delivery messages, hide +plain Retry and Edit, render **Open run**, and join the live status row by +`run_id`, `orchestration_run_id` and `step_id`. The Retry/Edit refusal checks +this key, and also checks the `assistant_workflow_delivery_` id prefix. + +### Delivered result answer + +A delivered composed or trimmed result has: + +- `metadata.workflow_delivery.kind: "result"`; +- `metadata.workflow_result`: the 6a public descriptor snapshot; +- `metadata.workflow_result_contexts`: contexts used by Follow up and masking; +- `metadata.token_usage`: usage values when the model returned them; +- `metadata.user_info.user_id`; +- `metadata.thread_info` with deterministic thread id and previous thread id; +- optional content-check metadata attached by the existing chat content checker. + +6b-2 reads `metadata.workflow_result` for Follow up and reads the live status +row's `actions.retry` to decide whether to show Retry on a delivery note or +card. The metadata alone never makes Retry available. + +### Fixed note kinds + +| Note kind | 6a lineage keys | Why | +| --- | --- | --- | +| `analysis` | `workflow_result` and `workflow_result_contexts` | Saved-analysis results have no excerpts, but Follow up and masking should work from the descriptor. | +| `failed` | None | No result text is available. | +| `cancelled` | None | No result text is available. | +| `skipped` | None | No result text is posted. | +| `status` | None | The result cannot be shown here or the gate is off. | +| `content_blocked` | None | The model or trimmed result was blocked, so no result text is posted. | + +The snapshot rule applies to every kind. If the workflow is later edited, +deleted, resumed or blocked by another active run, 6b-2 must use the status row +and route responses as the live truth. + +## Configuration + +| Setting or constant | Value or behavior | +| --- | --- | +| `enable_chat_workflow_results` (**Use Workflow Results In Chat**) | Off by default. Checked before seeding delivery and again before reading result text. Uses `is_chat_workflow_results_enabled_for_user(settings, user_roles)`. | +| `allow_user_workflows` | Required by route decorators, worker access checks and the 6a gate. | +| `require_member_of_workflow_user` | Enforced through roles captured at start; role revocations after start are not visible to the headless worker. | +| `enable_chat_orchestration_workflow_runs` | Phase 5 gate that allows a plan to start a saved workflow. | +| `CHAT_DELIVERY_VERSION` | `1` | +| `DELIVERY_MESSAGE_ID_PREFIX` | `assistant_workflow_delivery_` | +| `NOTIFICATION_TYPE` | `workflow_chat_delivery` | +| `WORKFLOW_SCOPE` | `personal` | +| `DELIVERY_GRACE_SECONDS` / `DEFAULT_DEADLINE_SECONDS` / `MAX_DEADLINE_SECONDS` | `86400` / `86400` / `86400` | +| `LEASE_SECONDS` / `SWEEP_INTERVAL_SECONDS` / `SWEEP_LOCK_SECONDS` | `300` / `30` / `120` | +| `SWEEP_TOP` / `SWEEP_MAX_RUNS` / `SWEEP_MAX_SECONDS` / `SWEEP_TS_GRACE_SECONDS` | `25` / `8` / `90` / `120` | +| `MAX_ATTEMPTS` / `BACKOFF_SECONDS` | `8` / `(30, 60, 120, 240, 480, 900, 900, 900)` | +| `DEFER_RETRY_SECONDS` / `DEFER_MAX_SECONDS` | `30` / `900` | +| `STREAM_META_FRESH_SECONDS` / `RECENT_USER_MESSAGE_SECONDS` | `90` / `600` | +| `HINT_QUEUE_MAX` / `HISTORY_MAX` | `256` / `5` | +| `COMPOSE_TEMPERATURE` / `COMPOSE_MAX_TOKENS` / `REPLY_MAX_CHARS` | `0.3` / `1500` / `12000` | +| `TRIMMED_FALLBACK_CHARS` / `EXCERPT_BUDGET_BYTES` | `2000` / `48 * 1024` | +| Status-route `CONVERSATION_LIMIT` / `GLOBAL_LIMIT` | `20` / `50` | +| Status-route `LIVE_READ_LIMIT` / `WORKFLOW_READ_LIMIT` | `10` / `20` | +| Status-route `RECENT_DELIVERY_SECONDS` | `600` | + +## File structure + +| File | Purpose | +| --- | --- | +| `application\single_app\functions_workflow_chat_delivery.py` | Pure delivery contract: constants, text builders, metadata, seed, merge, reconcile, message ids, notification keys and hint queue. | +| `application\single_app\functions_workflow_chat_delivery_worker.py` | Delivery worker: claim, re-checks, deferral, compose, content safety, unread mark, message create, notices, sweep and outcomes. | +| `application\single_app\functions_workflow_chat_delivery_status.py` | Owner-only status route payload, queries, live-read caps and row shaping for 6b-2. | +| `application\single_app\functions_orchestration_workflow_runs.py` | Seeds delivery at Phase 5 start and changes the started-run note only when every started run will post back. | +| `application\single_app\functions_workflow_runtime.py` | Accepts `chat_delivery` in the durable queue, finalizes it into the first run write, reconciles projections and signals delivery. | +| `application\single_app\functions_personal_workflows.py` | Guarded save for chat-started runs that preserves stored delivery records. | +| `application\single_app\functions_notifications.py` | Adds `workflow_chat_delivery` and idempotency pass-through for chat response notices. | +| `application\single_app\functions_conversation_unread.py` | Adds `mark_conversation_unread_guarded`. | +| `application\single_app\route_backend_conversations.py` | Refuses Retry and Edit on delivery messages. | +| `application\single_app\route_backend_orchestration.py` | Adds the V2 status route. | +| `application\single_app\background_tasks.py` | Registers the delivery loop in the background task host and scheduler host. | +| `application\single_app\config.py` | Tracks version `0.261.218`. | + +## Usage + +### Enable or configure + +1. Turn on **Enable Personal Workflows**. If **Require WorkflowUser App Role** is + on, assign the `WorkflowUser` role to the people who should use this. +2. Turn on **Run Workflows From Chat** under **Admin Settings > Orchestration** + so chat plans can start saved workflows. +3. Turn on **Use Workflow Results In Chat** under **Admin Settings > Workflow** + so chat-started workflow results can be posted back. +4. Make sure the workflow has **Durable execution** on. Non-durable runs are not + started by Phase 5 and are not delivery candidates. + +### What users see + +A user asks in a private chat, "Run my weekly digest now," with orchestration on. +After approval, the answer lists the workflow as started and says: + +```text +I'll post the results here when the run finishes. You can also follow it in the workflow's run history in Workflows. +``` + +Later, when the run completes, the server posts a new assistant message at the +end of that chat, with a label such as: + +```text +Results from `Weekly digest` ยท you asked on Sun Jan 4, 2026, 9:30 PM EST +``` + +The chat is marked unread and the user receives one bell notification. If the +chat was deleted, shared or no longer belongs to the requester, no message is +posted; the user receives one workflow notice that links to the run. If the +workflow was deleted or the runtime control is missing, the record closes +silently because the run link would be dead or unknowable. + +## Testing and validation + +When this documentation was written, these workflow chat delivery tests existed +under `functional_tests\`: + +- `test_workflow_chat_delivery_control_pins.py` +- `test_workflow_chat_delivery_loop.py` +- `test_workflow_chat_delivery_notice_and_unread.py` +- `test_workflow_chat_delivery_off_golden.py` +- `test_workflow_chat_delivery_refusals.py` +- `test_workflow_chat_delivery_save_guard.py` +- `test_workflow_chat_delivery_status_route.py` +- `test_workflow_chat_delivery_worker.py` + +Route-policy changes are covered by: + +- `functional_tests\route_tests\test_route_blueprint_policy_inventory.py` +- `functional_tests\route_tests\test_route_unauthenticated_policy_contract.py` + +The docs inventory was regenerated with `scripts\build_docs_inventory.py`, and +the documentation coverage and site-quality tests were run. Mutation testing +results for the delivery implementation are recorded in the pull request. + +## Known limitations and residual risks + +- A third consecutive storage read failure in the double-fault guarded-save path + can still wipe `chat_delivery`. In that case no message is posted, and the + status route reports the run as `not_applicable`. +- Mark-before-create has small windows: the chat may look unread just before the + message exists, a permanent create failure can leave a dangling unread mark, + and a chat deleted between mark and create can leave an orphaned message. +- A crash after unread is marked but before the message is created can cause a + second compose call and possibly a second content-safety incident. The first + composed text is not persisted. +- Version-1 runtime controls do not always have `deadline_at`. Their expiry is + computed from the admitted deadline; a run that finishes after `expires_at` + gets a ready notice rather than a chat post. +- Role revocations after start are not seen by the headless worker, because it + has no live browser session. It uses the roles captured at start, as durable + background execution already does. +- Owner-visible run history can show the raw `chat_delivery` record. That record + contains roles, model ids, lease ids and codes, but never result text. +- The workflow's own alert rules still fire for the same run, so a user may get + both a workflow alert and a chat post. +- Group workflow delivery, in-plan waiting for run completion, and a reusable + generic unread helper beyond this scenario are out of scope for this phase. + +## Deviations from the plan + +The implementation differs from the approved plan in these documented ways: + +- The delivered bell preview is fixed non-empty text (`Results from "X" are in your chat`), never result text. +- The worker marks the chat unread before creating the message, and keeps + `skip_if_read_since` for retries after a read. +- The plan-step-goal question fallback was dropped; the worker uses the original + user message or the fixed generic instruction. +- Prompt size is pre-checked before compose, and the excerpt budget steps down + from 48 KB to 24 KB, 12 KB and 6 KB. +- Attempt exhaustion before a claim can complete closes silently at 8 attempts. +- Notices use deterministic idempotency keys. An excerpt that still cannot fit + becomes a status note. +- The note kind is derived from the terminal run state. Expired records never + reopen. +- The delivery loop runs inside the Flask app context. Workflow reads in the + worker are direct container reads. +- Retry/Edit refusal payloads are `{error, code}` with HTTP 400. +- Status-route errors are `{error, code}`. `cancelling` and `waiting_recovery` + report as running. Unknown runtime states fall back to stored status and then + to `running` / `running`. +- `step_label` is always `null` in the status route. The workflow-read cap is + 20, separate from the 10 live runtime reads. +- `expired` applies only to non-terminal status rows. Status-route timestamps + have seconds precision. +- The definition revision for retry eligibility is computed from the editor form + exactly as durable resume does. Approve requires a `gate_id`. +- An invalid user id returns 503. `available: false` still returns stored rows. + An empty `conversation_id=` returns 400. +- Rebuilt or linked runs keep the older Phase 5 follow-up note text. +- The workflow notice link is the classic + `/workflow-activity?workflowId=...&runId=...&scope=personal` path. +- The V2 bell shows the generic `Notification` label for + `workflow_chat_delivery` until 6b-2 adds a specific case. + +## Related + +- [Chat orchestration workflow runs](CHAT_ORCHESTRATION_WORKFLOW_RUNS.md) +- [Workflow results in chat](CHAT_WORKFLOW_RESULTS_FOLLOW_UP.md) +- [Workflow settings](../../admin/workflow.md) +- [Orchestration settings](../../admin/orchestration.md) +- [Trigger a workflow](../../guides/trigger-a-workflow.md) +- [Ask about workflow results](../../guides/ask-about-workflow-results.md) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index e34eea96e..710077d74 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,17 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.218)** + +#### New Features + +* **Workflow results posted back to chat** + * When a chat orchestration plan starts one of the user's saved durable personal workflows, the server can now post the run's terminal outcome back into the same private chat after the run finishes, even hours later. + * The behavior is gated by **Use Workflow Results In Chat** (`enable_chat_workflow_results`), checked when the run starts and again when delivery happens, in addition to the existing Run Workflows From Chat gates. + * Delivery is exactly once per durable runtime generation: one deterministic chat message, one unread mark, and one bell notification. If the chat can no longer receive the post, the server posts nothing and sends one workflow notification with a run link instead. + * A new owner-only status route, `GET /api/v2/orchestration/workflow-runs/status`, gives 6b-2 the batched run-card contract. Retry/Edit on delivery messages are refused with fixed 400 codes. + * (Ref: #1546, #1543, `functions_workflow_chat_delivery.py`, `functions_workflow_chat_delivery_worker.py`, `functions_workflow_chat_delivery_status.py`, `functions_orchestration_workflow_runs.py`, `functions_workflow_runtime.py`, `route_backend_orchestration.py`, `route_backend_conversations.py`, [Workflow result delivery to chat](features/CHAT_WORKFLOW_RESULT_DELIVERY.md)) + ### **(v0.261.216)** #### Bug Fixes diff --git a/docs/guides/trigger-a-workflow.md b/docs/guides/trigger-a-workflow.md index 956c96be5..646400ec0 100644 --- a/docs/guides/trigger-a-workflow.md +++ b/docs/guides/trigger-a-workflow.md @@ -83,6 +83,12 @@ alerts. Stopping the plan doesn't stop a workflow it already started, so cancel from its run history if you need to. Retrying a failed step never starts a workflow twice: when the plan already started it, the retry links that run instead. +When your administrator also turns on **Use Workflow Results In Chat**, a +durable personal run started from chat can post its outcome back into that same +private chat after it finishes. The chat is marked unread, and a bell +notification opens the chat. If the chat was deleted or shared before delivery, +you get a workflow notification that opens the run instead. + A workflow that's already running isn't started again; the answer says so, and you can start it once that run finishes. If a run the plan started waits for a Microsoft 365 approval or sign-in, its status shows **Waiting**; follow From fea47130a9393994edfff7f8340cdcb514bd9845 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 04:00:20 -0400 Subject: [PATCH 13/36] Add placement and masking parity tests for delivered workflow results Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...low_chat_delivery_placement_and_masking.py | 336 ++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 functional_tests/test_workflow_chat_delivery_placement_and_masking.py diff --git a/functional_tests/test_workflow_chat_delivery_placement_and_masking.py b/functional_tests/test_workflow_chat_delivery_placement_and_masking.py new file mode 100644 index 000000000..e88a882ea --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_placement_and_masking.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_placement_and_masking.py +""" +Functional test for workflow chat delivery placement and masking parity. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures delivered workflow-result chat messages are placed like normal assistant replies and are masked on read exactly like 6a Follow up answers with the same lineage. +""" + +import copy +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +import functions_workflow_chat_delivery_worker as worker # noqa: E402 +from functions_workflow_chat_delivery import workflow_delivery_message_id # noqa: E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + RESULT_TEXT, + RUN_ID, + USER, + WORKFLOW_ID, + fake_response, + make_conversation, + make_invocation, + make_request_messages, + make_run, + make_workflow, + make_world, + parse_iso, + require, + shift, +) +from test_support.workflow_result_chat import ( # noqa: E402 + RUN_ID as RESULT_RUN_ID, + USER as RESULT_USER, + WORKFLOW_ID as RESULT_WORKFLOW_ID, +) +from test_support.workflow_result_offline_app import offline_workflow_result_app # noqa: E402 + + +DELIVERED_CANARY = "DELIVERED-WORKFLOW-RESULT-CANARY" +FOLLOWUP_CANARY = "FOLLOWUP-WORKFLOW-RESULT-CANARY" +LATER_USER_CANARY = "Can you compare those two workflow answers?" + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.218") + + +def deliver(world, *, user_id=USER, run_id=RUN_ID): + outcome = worker.process_workflow_chat_delivery(user_id, run_id, services=world.services) + require(outcome == worker.OUTCOME_DELIVERED, f"delivery did not complete: {outcome}") + messages = world.delivery_messages() + require(len(messages) == 1, f"expected one delivered message, got {messages}") + return messages[0] + + +def descriptor_result(descriptor, *, text=RESULT_TEXT): + def read_with_budget(excerpt_budget_bytes): + return { + "descriptor": copy.deepcopy(descriptor), + "excerpts": [{"text": text}], + "truncated": False, + "partial": False, + "analysis_only": False, + "skipped_reports": False, + "saved_inputs": [], + "budget": excerpt_budget_bytes, + } + + return read_with_budget + + +def make_offline_delivery_world(descriptor, *, reply=DELIVERED_CANARY): + run = make_run( + id=RESULT_RUN_ID, + user_id=RESULT_USER, + workflow_id=RESULT_WORKFLOW_ID, + workflow_name="Weekly digest", + chat_invocation=make_invocation(requested_by=RESULT_USER), + ) + world = make_world(run=run, control=False) + world.runtime.set(run_id=RESULT_RUN_ID, state="completed", version=5) + world.workflows.put(make_workflow(id=RESULT_WORKFLOW_ID, user_id=RESULT_USER, name="Weekly digest")) + world.conversations.put(make_conversation(user_id=RESULT_USER, chat_type="new", title="Workflow delivery chat")) + world.reader.outcomes.append(descriptor_result(descriptor)) + world.models.selected.replies.append(fake_response(reply)) + return world + + +def seed_chat(config, delivery_world, *extra_messages): + conversation = delivery_world.conversation() + conversation["user_id"] = RESULT_USER + conversation["chat_type"] = "new" + config.cosmos_conversations_container.upsert_item(conversation) + for message in [*make_request_messages(), *extra_messages]: + item = copy.deepcopy(message) + item["conversation_id"] = CONVERSATION_ID + config.cosmos_messages_container.upsert_item(item) + + +def six_a_followup_answer_like(delivered): + delivered_thread = delivered["metadata"]["thread_info"]["thread_id"] + metadata = { + "workflow_result": copy.deepcopy(delivered["metadata"]["workflow_result"]), + "workflow_result_contexts": copy.deepcopy(delivered["metadata"]["workflow_result_contexts"]), + "token_usage": {"total_tokens": 7}, + "user_info": {"user_id": RESULT_USER}, + "thread_info": { + "thread_id": "thread-six-a-followup-answer", + "previous_thread_id": delivered_thread, + "active_thread": True, + "thread_attempt": 1, + }, + "masked": False, + } + return { + "id": "assistant-six-a-followup-answer", + "conversation_id": CONVERSATION_ID, + "role": "assistant", + "content": FOLLOWUP_CANARY, + "timestamp": shift(delivered["timestamp"], microseconds=1), + "model_deployment_name": "gpt-4o", + "augmented": False, + "hybrid_citations": [], + "hybridsearch_query": None, + "agent_citations": [], + "web_search_citations": [], + "metadata": metadata, + } + + +def later_user_message(after_message): + return { + "id": "message-user-after-results", + "conversation_id": CONVERSATION_ID, + "role": "user", + "content": LATER_USER_CANARY, + "timestamp": shift(after_message["timestamp"], microseconds=1), + "metadata": { + "thread_info": { + "thread_id": "thread-user-after-results", + "previous_thread_id": after_message["metadata"]["thread_info"]["thread_id"], + "active_thread": True, + "thread_attempt": 1, + }, + }, + } + + +def response_messages(response, route_name): + require(response.status_code == 200, f"{route_name} failed: {response.status_code} {response.get_data(as_text=True)[:500]}") + payload = response.get_json() + require(isinstance(payload, dict) and isinstance(payload.get("messages"), list), f"{route_name} returned {payload}") + return payload["messages"] + + +def v2_messages(client): + return response_messages( + client.get("/api/get_messages", query_string={"conversation_id": CONVERSATION_ID}), + "V2 message loader", + ) + + +def classic_messages(client): + return response_messages(client.get(f"/conversation/{CONVERSATION_ID}/messages"), "classic message loader") + + +def search_count(client, term): + response = client.post("/api/search_conversations", json={"search_term": term}) + require(response.status_code == 200, f"search failed: {response.status_code} {response.get_data(as_text=True)[:500]}") + payload = response.get_json() + results = payload.get("results") if isinstance(payload, dict) else None + require(isinstance(results, list), f"search returned {payload}") + return sum( + len(result.get("messages") or []) + for result in results + if (result.get("conversation") or {}).get("id") == CONVERSATION_ID + ) + + +def history_text(world, monkeypatch): + monkeypatch.setattr(world.chats, "get_current_user_id", lambda: RESULT_USER) + messages = sorted( + [copy.deepcopy(item) for item in world.config.cosmos_messages_container.items.values()], + key=lambda item: item.get("timestamp") or "", + ) + with world.web.test_request_context("/"): + segments = world.chats.build_conversation_history_segments( + messages, + 20, + include_assistant_citation_context=False, + ) + return "\n".join(message.get("content", "") for message in segments["history_messages"]) + + +def require_surface_visibility(world, client, monkeypatch, *, delivered_visible, followup_visible, label): + loaded = v2_messages(client) + by_id = {message["id"]: message for message in loaded} + delivered = by_id[workflow_delivery_message_id(RESULT_RUN_ID, CONVERSATION_ID, 5)] + followup = by_id["assistant-six-a-followup-answer"] + require((DELIVERED_CANARY in delivered["content"]) is delivered_visible, f"{label}: V2 delivered mismatch") + require((FOLLOWUP_CANARY in followup["content"]) is followup_visible, f"{label}: V2 follow-up mismatch") + + search_matches = search_count(client, "WORKFLOW-RESULT-CANARY") + expected_matches = int(delivered_visible) + int(followup_visible) + require(search_matches == expected_matches, f"{label}: search had {search_matches}, expected {expected_matches}") + + context_text = history_text(world, monkeypatch) + require((DELIVERED_CANARY in context_text) is delivered_visible, f"{label}: history delivered mismatch") + require((FOLLOWUP_CANARY in context_text) is followup_visible, f"{label}: history follow-up mismatch") + + +def test_delivered_message_is_placed_after_the_newest_existing_thread_and_updates_the_conversation(): + world = make_world() + newest = world.messages.get("message-assistant-1") + + message = deliver(world) + + thread_info = message["metadata"]["thread_info"] + conversation = world.conversation() + require(thread_info["previous_thread_id"] == "thread-request", f"wrong previous thread: {thread_info}") + require(parse_iso(message["timestamp"]) > parse_iso(newest["timestamp"]), "delivery timestamp must follow newest") + require( + parse_iso(conversation["last_updated"]) >= parse_iso(message["timestamp"]), + f"conversation was not updated after delivery: {conversation}", + ) + require(thread_info["active_thread"] is True, f"delivery must be active: {thread_info}") + require(thread_info["thread_attempt"] == 1, f"delivery must not be a retry attempt: {thread_info}") + + +def test_delivered_result_shape_is_assistant_and_failed_notes_do_not_carry_result_lineage(): + result_world = make_world() + message = deliver(result_world) + metadata = message["metadata"] + + require(message["role"] == "assistant", f"unexpected role: {message}") + require(message["conversation_id"] == CONVERSATION_ID, f"unexpected conversation: {message}") + require(message["id"] == workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5), f"unexpected id: {message}") + require(metadata["workflow_delivery"]["workflow_id"] == WORKFLOW_ID, f"wrong workflow delivery: {metadata}") + require(metadata["workflow_delivery"]["run_id"] == RUN_ID, f"wrong workflow delivery: {metadata}") + require("workflow_result" in metadata, "result delivery must carry the 6a public descriptor") + require("workflow_result_contexts" in metadata, "result delivery must carry 6a lineage contexts") + + failed_world = make_world(run=make_run(status="failed"), state="failed") + failed_message = deliver(failed_world) + failed_metadata = failed_message["metadata"] + require("workflow_delivery" in failed_metadata, "failed note must keep delivery metadata") + for key in ("workflow_result", "workflow_result_context", "workflow_result_contexts"): + require(key not in failed_metadata, f"failed note must not carry {key}: {failed_metadata}") + + +def test_classic_and_v2_loaders_return_the_delivered_message_last_and_active(monkeypatch): + with offline_workflow_result_app() as world: + delivery_world = make_offline_delivery_world(world.fixture.read()["descriptor"]) + delivered = deliver(delivery_world, user_id=RESULT_USER, run_id=RESULT_RUN_ID) + seed_chat(world.config, delivery_world, delivered) + client = world.signed_in(RESULT_USER, "Owner") + + for route_name, loader in (("classic", classic_messages), ("V2", v2_messages)): + messages = loader(client) + ids = [message["id"] for message in messages] + require(ids[-1] == delivered["id"], f"{route_name} loader did not return delivery last: {ids}") + thread_info = messages[-1]["metadata"]["thread_info"] + require(thread_info["active_thread"] is True, f"{route_name} loader lost active thread: {thread_info}") + require(thread_info["thread_attempt"] == 1, f"{route_name} loader changed attempt: {thread_info}") + require(thread_info["previous_thread_id"] == "thread-request", f"{route_name} loader changed placement") + + +def test_masking_surfaces_show_delivered_result_and_matching_followup_while_source_is_intact(monkeypatch): + with offline_workflow_result_app() as world: + delivery_world = make_offline_delivery_world(world.fixture.read()["descriptor"]) + delivered = deliver(delivery_world, user_id=RESULT_USER, run_id=RESULT_RUN_ID) + followup = six_a_followup_answer_like(delivered) + seed_chat(world.config, delivery_world, delivered, followup, later_user_message(followup)) + client = world.signed_in(RESULT_USER, "Owner") + + require_surface_visibility( + world, + client, + monkeypatch, + delivered_visible=True, + followup_visible=True, + label="intact result", + ) + + +def test_masking_surfaces_withhold_delivered_result_and_matching_followup_after_run_deleted(monkeypatch): + with offline_workflow_result_app() as world: + delivery_world = make_offline_delivery_world(world.fixture.read()["descriptor"]) + delivered = deliver(delivery_world, user_id=RESULT_USER, run_id=RESULT_RUN_ID) + followup = six_a_followup_answer_like(delivered) + seed_chat(world.config, delivery_world, delivered, followup, later_user_message(followup)) + client = world.signed_in(RESULT_USER, "Owner") + world.fixture.containers["runs"].documents.clear() + + require_surface_visibility( + world, + client, + monkeypatch, + delivered_visible=False, + followup_visible=False, + label="deleted run", + ) + + +def test_masking_surfaces_withhold_delivered_result_and_matching_followup_after_result_changes(monkeypatch): + with offline_workflow_result_app() as world: + delivery_world = make_offline_delivery_world(world.fixture.read()["descriptor"]) + delivered = deliver(delivery_world, user_id=RESULT_USER, run_id=RESULT_RUN_ID) + followup = six_a_followup_answer_like(delivered) + seed_chat(world.config, delivery_world, delivered, followup, later_user_message(followup)) + client = world.signed_in(RESULT_USER, "Owner") + world.fixture.add_task("task-summary-72", {"reply": "A revised digest."}, order=2, label="Write the digest") + + require_surface_visibility( + world, + client, + monkeypatch, + delivered_visible=False, + followup_visible=False, + label="changed result", + ) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From b918f6400e4f3c704b4cca0b114680bc21529f61 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 04:10:11 -0400 Subject: [PATCH 14/36] Add projection and seed tests for workflow chat delivery Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...kflow_chat_delivery_projection_and_seed.py | 621 ++++++++++++++++++ 1 file changed, 621 insertions(+) create mode 100644 functional_tests/test_workflow_chat_delivery_projection_and_seed.py diff --git a/functional_tests/test_workflow_chat_delivery_projection_and_seed.py b/functional_tests/test_workflow_chat_delivery_projection_and_seed.py new file mode 100644 index 000000000..779a9290b --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_projection_and_seed.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_projection_and_seed.py +""" +Functional test for workflow chat delivery projection and seeding. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures chat-started workflow runs seed delivery records and runtime projection reconciles them without blocking durable progress. +""" + +import copy +import importlib +import sys +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import ModuleType, SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +import functions_workflow_chat_delivery as delivery # noqa: E402 +import functions_workflow_runtime as runtime # noqa: E402 +from functions_workflow_runtime_store import CONTROL_ID, CONTROL_TYPE, WorkflowRuntimeStore # noqa: E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + RUN_ID, + STEP_ID, + USER, + WORKFLOW_ID, + FakeContainer, + require, +) + + +NOW = datetime(2026, 5, 4, 15, 0, tzinfo=timezone.utc) +NOW_ISO = NOW.isoformat() +REQUEST_ID = "63e581a5-a430-4821-b1f4-1398e214fa53" + + +class Clock: + def __init__(self, now=NOW): + self.now = now + + def __call__(self): + return self.now + + def set(self, value): + self.now = value + + +def workflow(**overrides): + body = { + "id": WORKFLOW_ID, + "user_id": USER, + "name": "Daily digest", + "task_prompt": "Summarize the daily digest.", + "durable_execution": True, + "definition_version": 1, + "active_run_id": RUN_ID, + "active_runtime_version": 1, + "status": "queued", + } + body.update(overrides) + return body + + +def definition_revision(): + return runtime.workflow_definition_revision(workflow(active_run_id="", active_runtime_version=0, status="idle")) + + +def control(state="queued", version=1, *, schema_version=1, deleted=False, **overrides): + body = { + "id": CONTROL_ID, + "type": CONTROL_TYPE, + "item_type": CONTROL_TYPE, + "kind": "run", + "schema_version": schema_version, + "workflow_id": WORKFLOW_ID, + "user_id": USER, + "group_id": None, + "scope_type": "personal", + "scope_id": USER, + "run_id": RUN_ID, + "snapshot_ref": {"kind": "test-snapshot"}, + "definition_revision": definition_revision(), + "actor_user_id": USER, + "request_id": REQUEST_ID, + "created_at": NOW_ISO, + "updated_at": NOW_ISO, + "state": state, + "version": version, + "units": {}, + "memory": {}, + "gate": None, + "lease": None, + "deleted": deleted, + } + body.update(overrides) + return body + + +def chat_invocation(): + return { + "version": 1, + "source": delivery.CHAT_TRIGGER_SOURCE, + "conversation_id": CONVERSATION_ID, + "user_message_id": "message-user-1", + "orchestration_run_id": "orchestration-run-1", + "step_id": STEP_ID, + "requested_by": USER, + "requested_at": NOW_ISO, + } + + +def pending_record(control_doc=None): + seed = delivery.build_chat_delivery_seed( + time_zone="America/New_York", + model_selection={"model": {"model_deployment": "gpt-4o"}, "reasoning_effort": "medium"}, + requester_roles=["WorkflowUser"], + now=NOW, + ) + record = delivery.finalize_chat_delivery_seed(seed, control_doc or control()) + record["expires_at"] = "2099-01-01T00:00:00+00:00" + return record + + +def run_document(*, status="queued", chat_delivery=None, runtime_version=0, invocation=True): + body = { + "id": RUN_ID, + "workflow_id": WORKFLOW_ID, + "workflow_name": "Daily digest", + "user_id": USER, + "workspace_type": "personal", + "trigger_source": delivery.CHAT_TRIGGER_SOURCE, + "triggered_by": USER, + "durable_execution": True, + "status": status, + "success": False, + "definition_version": 1, + "started_at": NOW_ISO, + "completed_at": None, + "definition_revision": definition_revision(), + "runtime_version": runtime_version, + } + if invocation: + body["chat_invocation"] = chat_invocation() + if chat_delivery is not None: + body[delivery.CHAT_DELIVERY_KEY] = copy.deepcopy(chat_delivery) + return body + + +@pytest.fixture(autouse=True) +def clean_hints(): + delivery.clear_workflow_chat_delivery_hints() + yield + delivery.clear_workflow_chat_delivery_hints() + + +@pytest.fixture +def runtime_world(monkeypatch): + clock = Clock() + wf = workflow() + definitions = FakeContainer("definitions", [wf], clock=lambda: clock().isoformat()) + runs = FakeContainer("runs", clock=lambda: clock().isoformat()) + controls = FakeContainer("controls", clock=lambda: clock().isoformat()) + snapshots = {} + state = SimpleNamespace( + clock=clock, + workflow=wf, + definitions=definitions, + runs=runs, + controls=controls, + snapshots=snapshots, + ) + + def services(_workflow): + return { + "definitions": definitions, + "runs": runs, + "partition": USER, + "load_workflow": lambda: definitions.read_item(item=WORKFLOW_ID, partition_key=USER), + "settings": lambda: {"allow_user_workflows": True}, + } + + def make_store(current_workflow, run_id): + return WorkflowRuntimeStore(controls, current_workflow, run_id, clock=clock) + + def save_result(_workflow, run_id, task_id, result, **_kwargs): + reference = {"kind": "test-result", "run_id": run_id, "task_id": task_id} + snapshots[(run_id, task_id)] = copy.deepcopy(result) + return reference + + def load_result(_workflow, run_id, task_id, reference): + if isinstance(reference, dict) and (reference.get("run_id"), reference.get("task_id")) in snapshots: + return copy.deepcopy(snapshots[(reference["run_id"], reference["task_id"])]) + return copy.deepcopy(wf) + + monkeypatch.setattr(runtime, "_services", services) + monkeypatch.setattr(runtime, "_authorize_execution", lambda *_args, **_kwargs: None) + monkeypatch.setattr(runtime, "workflow_runtime_store", make_store) + monkeypatch.setattr(runtime, "save_workflow_task_result", save_result) + monkeypatch.setattr(runtime, "load_workflow_task_result", load_result) + monkeypatch.setattr(runtime, "delete_workflow_run_results", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + runtime, + "workflow_runtime_status", + lambda current_workflow, run_id, *, reader_user_id: runtime.workflow_runtime_projection( + make_store(current_workflow, run_id).read() + ), + ) + return state + + +def put_control(world, body): + world.controls.put(body) + return copy.deepcopy(body) + + +def put_run(world, body): + world.runs.put(body) + return copy.deepcopy(body) + + +def stored_run(world): + return world.runs.read_item(item=RUN_ID, partition_key=USER) + + +def project(world, control_doc, *, result=None): + put_control(world, control_doc) + return runtime._project_runtime_run( # noqa: SLF001 - functional test pins projection contract. + {"definitions": world.definitions, "runs": world.runs, "partition": USER, "load_workflow": lambda: world.workflow}, + world.workflow, + RUN_ID, + control_doc, + result=result, + ) + + +def terminal_result(state): + return { + "run": { + "status": state, + "success": state in {"completed", "completed_partial"}, + "completed_at": "2026-05-04T15:05:00+00:00", + }, + } + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.218") + + +def test_projection_reconciles_terminal_states_and_signals_once(runtime_world): + cases = [ + ("completed", delivery.KIND_RESULT), + ("failed", delivery.KIND_FAILED), + ("cancelled", delivery.KIND_CANCELLED), + ] + for index, (state, expected_kind) in enumerate(cases, start=1): + delivery.clear_workflow_chat_delivery_hints() + runtime_world.controls.items.clear() + runtime_world.runs.items.clear() + control_doc = control(state=state, version=10 + index) + put_run(runtime_world, run_document(chat_delivery=pending_record(control_doc))) + project(runtime_world, control_doc, result=terminal_result(state)) + saved = stored_run(runtime_world) + record = saved[delivery.CHAT_DELIVERY_KEY] + require(record["status"] == delivery.STATUS_READY, f"{state} should make chat delivery ready") + require(record["kind"] == expected_kind, f"{state} should map to {expected_kind}") + require(record["generation"] == control_doc["version"], f"{state} should use the control version") + hints = delivery.drain_workflow_chat_delivery_hints() + require(hints == [(USER, RUN_ID)], f"{state} should queue exactly one in-process delivery hint") + + delivery.clear_workflow_chat_delivery_hints() + runtime_world.controls.items.clear() + runtime_world.runs.items.clear() + waiting = control(state="waiting_output", version=30) + put_run(runtime_world, run_document(chat_delivery=pending_record(waiting))) + project(runtime_world, waiting) + saved = stored_run(runtime_world) + require(saved[delivery.CHAT_DELIVERY_KEY]["status"] == delivery.STATUS_PENDING, "waiting runs must stay pending") + hints = delivery.drain_workflow_chat_delivery_hints() + require(hints == [], "waiting runs must not queue a delivery hint") + + +def test_projection_keeps_stored_chat_delivery_before_reconciling(runtime_world): + control_doc = control(state="completed", version=7) + stored_record = pending_record(control_doc) + stored_record["time_zone"] = "America/Los_Angeles" + stale_record = pending_record(control_doc) + stale_record["time_zone"] = "UTC" + put_run(runtime_world, run_document(chat_delivery=stored_record)) + result = terminal_result("completed") + result["run"][delivery.CHAT_DELIVERY_KEY] = stale_record + project(runtime_world, control_doc, result=result) + record = stored_run(runtime_world)[delivery.CHAT_DELIVERY_KEY] + require(record["time_zone"] == "America/Los_Angeles", "stored chat_delivery must win over result payload copies") + require(record["status"] == delivery.STATUS_READY, "the preserved stored record should still be reconciled") + + +def test_projection_without_chat_delivery_does_not_create_record_or_hint(runtime_world): + control_doc = control(state="completed", version=4) + put_run(runtime_world, run_document(invocation=False)) + project(runtime_world, control_doc, result=terminal_result("completed")) + saved = stored_run(runtime_world) + require(delivery.CHAT_DELIVERY_KEY not in saved, "non-chat-started runs must not gain chat_delivery") + hints = delivery.drain_workflow_chat_delivery_hints() + require(hints == [], "non-chat-started runs must not queue a delivery hint") + + +def test_projection_survives_reconcile_failure_and_still_writes_runtime(monkeypatch, runtime_world): + control_doc = control(state="completed", version=8) + original_record = pending_record(control_doc) + put_run(runtime_world, run_document(chat_delivery=original_record)) + + def fail_reconcile(*_args, **_kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr(runtime, "reconcile_chat_delivery", fail_reconcile) + projected = project(runtime_world, control_doc, result=terminal_result("completed")) + saved = stored_run(runtime_world) + require(projected["status"] == "completed", "projection should return normally after reconcile failure") + require(saved["status"] == "completed", "projection should still write terminal runtime status") + require(saved["runtime_version"] == control_doc["version"], "projection should still write runtime version") + require(saved[delivery.CHAT_DELIVERY_KEY] == original_record, "reconcile failure should leave stored record unchanged") + + +def test_projection_survives_broken_hint_queue_and_direct_signal_returns_false(monkeypatch, runtime_world): + class BrokenLock: + def __enter__(self): + raise RuntimeError("lock unavailable") + + def __exit__(self, *_args): + return False + + control_doc = control(state="completed", version=9) + put_run(runtime_world, run_document(chat_delivery=pending_record(control_doc))) + monkeypatch.setattr(delivery, "_HINT_LOCK", BrokenLock()) + direct = delivery.signal_workflow_chat_delivery(USER, RUN_ID) + require(direct is False, "direct signal should fail closed when hint internals raise") + projected = project(runtime_world, control_doc, result=terminal_result("completed")) + saved = stored_run(runtime_world) + record = saved[delivery.CHAT_DELIVERY_KEY] + require(projected["status"] == "completed", "projection should return normally when hint signaling fails") + require(record["status"] == delivery.STATUS_READY, "ready record must be persisted even when signal fails") + require(record["next_attempt_at"] is None, "a later sweep can find the ready record immediately") + + +def test_resume_reopens_delivered_generation_and_expired_never_reopens(runtime_world): + failed = control(state="failed", version=5) + delivered = pending_record(failed) + delivered.update({ + "status": delivery.STATUS_DELIVERED, + "generation": 5, + "kind": delivery.KIND_FAILED, + "message_id": "assistant-delivery-1", + "delivered_at": "2026-05-04T15:10:00+00:00", + }) + put_control(runtime_world, failed) + put_run(runtime_world, run_document(status="failed", chat_delivery=delivered, runtime_version=5)) + request_id = str(uuid.uuid4()) + runtime.decide_workflow_runtime( + runtime_world.workflow, + RUN_ID, + {"expected_version": 5, "request_id": request_id}, + actor_user_id=USER, + resume=True, + ) + reopened = stored_run(runtime_world)[delivery.CHAT_DELIVERY_KEY] + require(reopened["status"] == delivery.STATUS_PENDING, "resume should reopen a delivered previous generation") + require(reopened["generation"] is None, "reopened records wait for the next terminal generation") + require(len(reopened["history"]) == 1, "resume should archive the previous delivered outcome") + require(reopened["history"][0]["generation"] == 5, "history should preserve the delivered generation") + + runtime_world.controls.items.clear() + runtime_world.runs.items.clear() + later = control(state="completed", version=8) + expired = pending_record(later) + expired.update({"status": delivery.STATUS_EXPIRED, "generation": 5, "kind": delivery.KIND_EXPIRED}) + put_run(runtime_world, run_document(chat_delivery=expired, runtime_version=5)) + project(runtime_world, later, result=terminal_result("completed")) + record = stored_run(runtime_world)[delivery.CHAT_DELIVERY_KEY] + require(record["status"] == delivery.STATUS_EXPIRED, "expired records must not reopen on a later projection") + require(record["generation"] == 5, "expired records should preserve their closed generation") + + +def test_deleted_control_closes_delivery_as_workflow_deleted_without_hint(runtime_world): + deleted = control(state="cancelled", version=3, deleted=True) + body = run_document(chat_delivery=pending_record(deleted)) + ready = runtime._reconcile_chat_delivery(body, deleted, RUN_ID) # noqa: SLF001 - pins runtime reconcile wrapper. + record = body[delivery.CHAT_DELIVERY_KEY] + require(ready is False, "deleted controls should not signal delivery") + require(record["status"] == delivery.STATUS_UNDELIVERABLE, "deleted controls should close as undeliverable") + require(record["outcome_reason"] == delivery.REASON_WORKFLOW_DELETED, "deleted controls should use workflow_deleted") + hints = delivery.drain_workflow_chat_delivery_hints() + require(hints == [], "deleted controls should not queue hints") + + +def test_deadline_expiration_projects_expired_kind(runtime_world): + deadline = (NOW - timedelta(minutes=1)).isoformat() + schema_two = control( + state="queued", + version=2, + schema_version=2, + cursor={"region_id": "flow", "node_id": "node-1"}, + journal_counts={}, + completed_unit_count=0, + admitted_count=0, + max_executions=10, + deadline_seconds=60, + deadline_at=deadline, + snapshot_identity={"node_id": "flow", "execution_id": "execution-1", "attempt": 1, "iteration_path": []}, + ) + put_control(runtime_world, schema_two) + put_run(runtime_world, run_document(chat_delivery=pending_record(schema_two))) + store = WorkflowRuntimeStore(runtime_world.controls, runtime_world.workflow, RUN_ID, clock=runtime_world.clock) + expired_control = store.expire_deadline() + project(runtime_world, expired_control) + record = stored_run(runtime_world)[delivery.CHAT_DELIVERY_KEY] + require(expired_control["state"] == "paused", "deadline expiry should pause the schema-v2 control") + require(record["status"] == delivery.STATUS_READY, "deadline-expired controls should be ready for notice delivery") + require(record["kind"] == delivery.KIND_EXPIRED, "deadline-expired controls should project the expired kind") + require(record["generation"] == expired_control["version"], "deadline-expired generation should match control version") + + +def test_chat_delivery_seed_gates_fields_and_logs_warning(monkeypatch): + wr = importlib.import_module("functions_orchestration_workflow_runs") + settings_module = ModuleType("functions_settings") + settings_module.is_chat_workflow_results_enabled_for_user = lambda settings, user_roles=None: True + monkeypatch.setitem(sys.modules, "functions_settings", settings_module) + logs = [] + monkeypatch.setattr(wr, "_log", lambda *args, **kwargs: logs.append((args, kwargs))) + context = SimpleNamespace( + user_roles=["WorkflowUser", "WorkflowUser", "", "Admin"], + time_zone="America/New_York", + seeds={ + "model": { + "model_deployment": "gpt-4o", + "model_provider": "aoai", + "model_endpoint_id": "endpoint-1", + "model_id": "model-1", + }, + "reasoning_effort": "high", + "active_group_ids": ["group-1", "group-1", "group-2"], + }, + active_group_ids=["fallback-group"], + run_id="orchestration-run-1", + ) + + closed = wr._chat_delivery_seed({}, context, {"time_zone": "UTC"}) # noqa: SLF001 + require(closed is None, "closed workflow-results setting should not create a seed") + off = wr._chat_delivery_seed({"enable_chat_workflow_results": False}, context, {"time_zone": "UTC"}) # noqa: SLF001 + require(off is None, "6a results gate off should not create a seed") + seed = wr._chat_delivery_seed({"enable_chat_workflow_results": True}, context, {"time_zone": "UTC"}) # noqa: SLF001 + require(seed["time_zone"] == "America/New_York", "context time zone should win when present") + require(seed["model_selection"]["model"]["model_deployment"] == "gpt-4o", "model deployment should be normalized") + require(seed["model_selection"]["reasoning_effort"] == "high", "reasoning effort should be normalized") + require(seed["model_selection"]["active_group_ids"] == ["group-1", "group-2"], "active groups should be deduped") + require(seed["requester_roles"] == ["WorkflowUser", "Admin"], "requester roles should be normalized") + + fallback_context = SimpleNamespace(user_roles=[], time_zone=None, seeds={}, active_group_ids=[]) + fallback = wr._chat_delivery_seed( # noqa: SLF001 + {"enable_chat_workflow_results": True}, + fallback_context, + {"time_zone": "Europe/Berlin"}, + ) + require(fallback["time_zone"] == "Europe/Berlin", "planning time zone should be used as fallback") + + def fail_gate(_settings, user_roles=None): + raise RuntimeError("gate failed") + + monkeypatch.setattr(settings_module, "is_chat_workflow_results_enabled_for_user", fail_gate) + failed = wr._chat_delivery_seed({"enable_chat_workflow_results": True}, context, {"time_zone": "UTC"}) # noqa: SLF001 + require(failed is None, "seed gate exceptions should fail closed") + require(logs, "seed gate exceptions should be warning-logged") + + +def test_queue_durable_workflow_run_finalizes_chat_delivery_seed(monkeypatch, runtime_world): + monkeypatch.setattr(runtime, "_now", lambda: NOW_ISO) + runtime_world.definitions.items.clear() + runtime_world.definitions.put(workflow(active_run_id="", active_runtime_version=0, status="idle")) + seed = pending_record(control()) + queued = runtime.queue_durable_workflow_run( + {"id": WORKFLOW_ID, "user_id": USER}, + actor_user_id=USER, + request_id=REQUEST_ID, + chat_invocation=chat_invocation(), + chat_delivery=seed, + ) + record = queued["run"][delivery.CHAT_DELIVERY_KEY] + require(record["status"] == delivery.STATUS_PENDING, "queued chat delivery should start pending") + require(isinstance(record["expires_at"], str) and record["expires_at"], "queued chat delivery should have expires_at") + require(queued["run"]["chat_invocation"]["requested_at"] == NOW_ISO, "chat invocation should preserve requested_at") + + other_world = runtime_world + other_world.controls.items.clear() + other_world.runs.items.clear() + other_world.definitions.items.clear() + other_world.definitions.put(workflow(active_run_id="", active_runtime_version=0)) + without = runtime.queue_durable_workflow_run( + {"id": WORKFLOW_ID, "user_id": USER}, + actor_user_id=USER, + request_id=str(uuid.uuid4()), + chat_invocation=chat_invocation(), + ) + require(delivery.CHAT_DELIVERY_KEY not in without["run"], "queue without a seed must not add chat_delivery") + + +def test_start_sets_chat_delivery_sidecar_only_when_seed_applies(monkeypatch): + wr = importlib.import_module("functions_orchestration_workflow_runs") + settings_module = ModuleType("functions_settings") + settings_module.is_user_workflows_enabled_for_user = lambda settings, user_roles=None: True + settings_module.is_chat_workflow_results_enabled_for_user = lambda settings, user_roles=None: True + monkeypatch.setitem(sys.modules, "functions_settings", settings_module) + monkeypatch.setattr(wr, "_read_conversation", lambda conversation_id: {"id": conversation_id, "user_id": USER}) + monkeypatch.setattr(wr, "refresh_workflow_planning_privacy", lambda planning, conversation, user_id: planning) + monkeypatch.setattr(wr, "_read_workflow", lambda user_id, workflow_id: workflow(id=workflow_id, active_run_id="")) + monkeypatch.setattr(wr, "_read_workflow_run", lambda user_id, run_id: None) + step = {"step_id": STEP_ID, "capability_id": wr.CAPABILITY_WORKFLOW_RUN, "arguments": {"workflow": "digest"}} + planning = { + "conversation_private": True, + "time_zone": "America/New_York", + "workflow_runs": {"ready": True}, + "handles": {"workflows": {"digest": {"id": WORKFLOW_ID, "name": "Daily digest"}}}, + "catalog": {"workflows": [{"handle": "digest", "name": "Daily digest"}]}, + } + context = SimpleNamespace( + conversation_id=CONVERSATION_ID, + workflow_planning=planning, + signed_in_session=True, + user_roles=["WorkflowUser"], + run_id="orchestration-run-1", + attempt_root_run_id="orchestration-run-1", + user_message_id="message-user-1", + seeds={}, + active_group_ids=[], + time_zone="America/New_York", + ) + queued_options = [] + + def queue_run(_workflow, **options): + queued_options.append(copy.deepcopy(options)) + run = run_document(chat_delivery=delivery.finalize_chat_delivery_seed(options["chat_delivery"], control())) + run["chat_invocation"] = copy.deepcopy(options["chat_invocation"]) + return {"run": run} + + monkeypatch.setattr(wr, "_queue_workflow_run", queue_run) + settings = { + "enable_chat_orchestration": True, + "enable_chat_orchestration_workflow_runs": True, + "allow_user_workflows": True, + "enable_chat_workflow_results": True, + "chat_orchestration_enabled_capabilities": [wr.CAPABILITY_WORKFLOW_RUN], + } + outcome = wr._start(step, context, settings=settings, user_id=USER, recheck=lambda: None, requested_at=NOW_ISO) # noqa: SLF001 + sidecar = wr._sidecar(step, context, user_id=USER, producer_run_id=context.run_id, outcome=outcome) # noqa: SLF001 + require(outcome.get("chat_delivery") is True, f"start outcome should flag applicable chat delivery: {outcome!r}") + require(sidecar.get("chat_delivery") is True, "sidecar should flag applicable chat delivery") + require(delivery.CHAT_DELIVERY_KEY in queued_options[0], "start should pass the seed when the 6a gate is on") + + queued_options.clear() + off_settings = dict(settings, enable_chat_workflow_results=False) + outcome_off = wr._start(step, context, settings=off_settings, user_id=USER, recheck=lambda: None, requested_at=NOW_ISO) # noqa: SLF001 + sidecar_off = wr._sidecar(step, context, user_id=USER, producer_run_id=context.run_id, outcome=outcome_off) # noqa: SLF001 + require(outcome_off.get("chat_delivery") is not True, "start outcome must not flag delivery when 6a gate is off") + require("chat_delivery" not in sidecar_off, "sidecar must not flag delivery when 6a gate is off") + require(delivery.CHAT_DELIVERY_KEY not in queued_options[0], "start must not pass a seed when the 6a gate is off") + + +def test_workflow_run_note_uses_delivery_follow_up_texts(): + wr = importlib.import_module("functions_orchestration_workflow_runs") + one_plan = { + "steps": [{"step_id": "run-one", "capability_id": wr.CAPABILITY_WORKFLOW_RUN, "arguments": {"workflow": "digest"}}], + } + one_record = [{ + "step_id": "run-one", + "capability_id": wr.CAPABILITY_WORKFLOW_RUN, + "status": wr.STEP_STATUS_COMPLETED, + "workflow_run": {"status": wr.WORKFLOW_RUN_STATUS_QUEUED, "name": "Daily digest", "chat_delivery": True}, + }] + one = wr.workflow_run_note(one_plan, one_record) + require(wr.WORKFLOW_RUN_DELIVERY_FOLLOW_UP in one, "one delivered run should use the delivery follow-up") + + many_plan = { + "steps": [ + {"step_id": "run-one", "capability_id": wr.CAPABILITY_WORKFLOW_RUN, "arguments": {"workflow": "digest"}}, + {"step_id": "run-two", "capability_id": wr.CAPABILITY_WORKFLOW_RUN, "arguments": {"workflow": "digest2"}}, + ], + } + many_records = [ + { + "step_id": "run-one", + "capability_id": wr.CAPABILITY_WORKFLOW_RUN, + "status": wr.STEP_STATUS_COMPLETED, + "workflow_run": {"status": wr.WORKFLOW_RUN_STATUS_QUEUED, "name": "Daily digest", "chat_delivery": True}, + }, + { + "step_id": "run-two", + "capability_id": wr.CAPABILITY_WORKFLOW_RUN, + "status": wr.STEP_STATUS_COMPLETED, + "workflow_run": {"status": wr.WORKFLOW_RUN_STATUS_QUEUED, "name": "Weekly digest", "chat_delivery": True}, + }, + ] + many = wr.workflow_run_note(many_plan, many_records) + require(wr.WORKFLOW_RUN_DELIVERY_FOLLOW_UP_MANY in many, "many delivered runs should use the many delivery follow-up") + + without_delivery_records = copy.deepcopy(one_record) + without_delivery_records[0]["workflow_run"].pop("chat_delivery") + without = wr.workflow_run_note(one_plan, without_delivery_records) + require(wr.WORKFLOW_RUN_FOLLOW_UP in without, "runs without delivery should use the original follow-up") + require(wr.WORKFLOW_RUN_DELIVERY_FOLLOW_UP not in without, "original follow-up should be byte-for-byte distinct") + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From cd375c80ef5e1ce136fd5f51d3aee22bba7fdd8b Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 04:10:12 -0400 Subject: [PATCH 15/36] Remove the reused offline session directory after placement tests app.py binds sessions to the first offline app's directory, so the later offline apps in one process recreate it after the first teardown removes it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...workflow_chat_delivery_placement_and_masking.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/functional_tests/test_workflow_chat_delivery_placement_and_masking.py b/functional_tests/test_workflow_chat_delivery_placement_and_masking.py index e88a882ea..ca007394e 100644 --- a/functional_tests/test_workflow_chat_delivery_placement_and_masking.py +++ b/functional_tests/test_workflow_chat_delivery_placement_and_masking.py @@ -9,6 +9,7 @@ """ import copy +import shutil import sys from pathlib import Path @@ -52,6 +53,19 @@ LATER_USER_CANARY = "Can you compare those two workflow answers?" +@pytest.fixture(scope="module", autouse=True) +def remove_reused_session_directory(): + # app.py binds sessions to the first offline app's directory, so the later offline apps here recreate it. + yield + application = getattr(sys.modules.get("app"), "app", None) + session_dir = application.config.get("SESSION_FILE_DIR") if application is not None else None + if not session_dir: + return + path = Path(session_dir).resolve() + if path.parent == ROOT / "functional_tests" and path.name.startswith(".workflow-result-chat-state-"): + shutil.rmtree(path, ignore_errors=True) + + def test_version_is_at_least_the_chat_delivery_release(): assert_app_version_at_least("0.261.218") From ad867bbd72acd696a45b7bd3ccd77b102e9d11d4 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 04:10:59 -0400 Subject: [PATCH 16/36] Add a 409 race test for delivery message create Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test_workflow_chat_delivery_worker.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/functional_tests/test_workflow_chat_delivery_worker.py b/functional_tests/test_workflow_chat_delivery_worker.py index 69e2a1d2a..70eac3f10 100644 --- a/functional_tests/test_workflow_chat_delivery_worker.py +++ b/functional_tests/test_workflow_chat_delivery_worker.py @@ -744,6 +744,28 @@ def create_then_fail(body=None, **kwargs): assert world.notifications.chat_calls == [delivered_notice()] +def test_a_create_that_loses_a_race_keeps_the_message_already_posted(): + world = make_world() + original_create = world.messages.create_item + + def another_attempt_posts_first(body=None, **kwargs): + world.messages.create_item = original_create + world.messages.put({**body, "content": "posted by a racing attempt"}) + return original_create(body=body, **kwargs) + + world.messages.create_item = another_attempt_posts_first + + outcome = deliver(world) + assert outcome == worker.OUTCOME_DELIVERED, "a 409 on create means the message is already in the chat" + + messages = world.delivery_messages() + assert [message["id"] for message in messages] == [M5] + assert messages[0]["content"] == "posted by a racing attempt", "the message already posted is kept" + assert world.record()["phase"] == "notified" + assert world.record()["attempts"] == 1, "a create that lost a race needs no retry" + assert world.notifications.chat_calls == [delivered_notice()] + + def test_a_failed_notice_is_sent_on_the_retry_without_reposting(): world = make_world() world.notifications.chat_errors.append(RuntimeError("bell down")) From 55566debf883e48e12b3bc3d48965c7afe1f8a5a Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 04:22:32 -0400 Subject: [PATCH 17/36] Add workflow chat delivery contract tests Pins the pure contract module: message and thread identity per generation, idempotency key formats, labels and notes, the metadata snapshot, backoff, seeds, and the reconcile reopen, tombstone, missing-control and expiry rules. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test_workflow_chat_delivery_contract.py | 791 ++++++++++++++++++ 1 file changed, 791 insertions(+) create mode 100644 functional_tests/test_workflow_chat_delivery_contract.py diff --git a/functional_tests/test_workflow_chat_delivery_contract.py b/functional_tests/test_workflow_chat_delivery_contract.py new file mode 100644 index 000000000..4522ce99b --- /dev/null +++ b/functional_tests/test_workflow_chat_delivery_contract.py @@ -0,0 +1,791 @@ +#!/usr/bin/env python3 +# test_workflow_chat_delivery_contract.py +""" +Functional test for the workflow chat delivery contract module. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures the pure delivery helpers preserve idempotent chat-result message contracts. +""" + +import copy +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for _path in (ROOT / "application" / "single_app", ROOT / "functional_tests"): + if str(_path) not in sys.path: + sys.path.insert(0, str(_path)) + +import pytest # noqa: E402 + +import functions_workflow_chat_delivery as delivery # noqa: E402 +from functions_workflow_result_reader import format_workflow_run_time # noqa: E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 +from test_support.workflow_chat_delivery_fakes import ( # noqa: E402 + CONVERSATION_ID, + RUN_ID, + STEP_ID, + USER, + WORKFLOW_ID, + require, +) + +NOW = datetime(2026, 5, 4, 15, 0, 0, tzinfo=timezone.utc) +NOW_TEXT = "2026-05-04T15:00:00.000000Z" +REQUESTED_AT = "2026-05-04T14:05:00.000000Z" + + +def _summary(state="completed", version=5, **overrides): + summary = { + "exists": True, + "deleted": False, + "version": version, + "state": state, + "phase": None, + "gate_reason_code": None, + "deadline_at": None, + "deadline_seconds": None, + "schema_version": 2, + } + summary.update(overrides) + return summary + + +def _record(status=delivery.STATUS_PENDING, generation=None, **overrides): + record = delivery.build_chat_delivery_seed( + time_zone="America/New_York", + model_selection={"model": {"model_deployment": "gpt-4o"}, "reasoning_effort": "medium"}, + requester_roles=["User"], + now=NOW, + ) + record = delivery.finalize_chat_delivery_seed(record, None) + record.update({"status": status, "generation": generation, "updated_at": NOW_TEXT}) + record.update(overrides) + return record + + +def _reconcile(record, summary, now=NOW): + return delivery.reconcile_chat_delivery(record, summary, now=now) + + +@pytest.fixture(autouse=True) +def clear_delivery_hints(): + delivery.clear_workflow_chat_delivery_hints() + yield + delivery.clear_workflow_chat_delivery_hints() + + +def test_version_is_at_least_the_chat_delivery_release(): + assert_app_version_at_least("0.261.218") + + +def test_workflow_delivery_message_id_is_deterministic_and_prefixed(): + first = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + second = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + require(first == second, "message id should be deterministic for the same run, chat, and generation") + require(first.startswith(delivery.DELIVERY_MESSAGE_ID_PREFIX), "message id should carry the delivery prefix") + require(len(first) == len(delivery.DELIVERY_MESSAGE_ID_PREFIX) + 40, "message id should use a 40-char fingerprint") + + +def test_workflow_delivery_message_id_changes_when_generation_changes(): + old_id = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + new_id = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 6) + require(old_id != new_id, "message id must include generation in its fingerprint") + + +def test_workflow_delivery_message_id_changes_when_run_or_conversation_changes(): + first = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + other_run = delivery.workflow_delivery_message_id(f"{RUN_ID}-other", CONVERSATION_ID, 5) + other_conversation = delivery.workflow_delivery_message_id(RUN_ID, f"{CONVERSATION_ID}-other", 5) + require(first != other_run, "message id should change when only the run id changes") + require(first != other_conversation, "message id should change when only the conversation id changes") + + +def test_workflow_delivery_message_id_rejects_missing_inputs(): + with pytest.raises(ValueError, match="run_id is required"): + delivery.workflow_delivery_message_id("", CONVERSATION_ID, 5) + with pytest.raises(ValueError, match="conversation_id is required"): + delivery.workflow_delivery_message_id(RUN_ID, "", 5) + with pytest.raises(ValueError, match="generation must be a non-negative integer"): + delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, -1) + with pytest.raises(ValueError, match="generation must be a non-negative integer"): + delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, True) + + +def test_delivery_thread_id_is_deterministic_and_distinct_per_message(): + first_message = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + second_message = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 6) + first_thread = delivery.delivery_thread_id(first_message) + repeat_thread = delivery.delivery_thread_id(first_message) + second_thread = delivery.delivery_thread_id(second_message) + require(first_thread == repeat_thread, "thread id should be deterministic for one message id") + require(first_thread != second_thread, "different message ids should get different thread ids") + with pytest.raises(ValueError, match="message_id is required"): + delivery.delivery_thread_id(" ") + + +def test_is_workflow_delivery_message_accepts_metadata_and_delivery_id(): + metadata_message = {"metadata": {delivery.DELIVERY_METADATA_KEY: {"run_id": RUN_ID}}} + id_message = {"id": delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5), "metadata": {}} + require(delivery.is_workflow_delivery_message(metadata_message), "metadata key should identify a workflow delivery message") + require(delivery.is_workflow_delivery_message(id_message), "delivery id prefix should identify a workflow delivery message") + + +def test_is_workflow_delivery_message_rejects_ordinary_and_malformed_messages(): + samples = [ + None, + [], + {}, + {"id": "assistant_regular_1", "metadata": {}}, + {"metadata": None}, + {"metadata": "not-a-dict"}, + {"metadata": {delivery.DELIVERY_METADATA_KEY: "not-a-dict"}}, + ] + for sample in samples: + result = delivery.is_workflow_delivery_message(sample) + require(result is False, f"malformed message should not be recognized: {sample!r}") + + +def test_delivery_notification_key_formats_are_pinned_literals(): + key = delivery.delivery_notification_key(RUN_ID, 5) + notice_key = delivery.undeliverable_notification_key(RUN_ID, 5) + expired_key = delivery.expired_notification_key(RUN_ID) + require(key == "workflow-chat-delivery:run-delivery-1:5", "delivery notification key format changed") + require(notice_key == "workflow-chat-delivery-notice:run-delivery-1:5", "undeliverable notification key format changed") + require(expired_key == "workflow-chat-delivery-notice:run-delivery-1:expired", "expired notification key format changed") + + +def test_token_usage_idempotency_key_format_is_pinned_literal(): + message_id = delivery.workflow_delivery_message_id(RUN_ID, CONVERSATION_ID, 5) + key = delivery.token_usage_idempotency_key(message_id, 3) + require(key == f"workflow_result_delivery:{message_id}:3", "token usage idempotency key format changed") + + +def test_idempotency_keys_differ_by_generation_and_attempt(): + gen_five = delivery.delivery_notification_key(RUN_ID, 5) + gen_six = delivery.delivery_notification_key(RUN_ID, 6) + attempt_one = delivery.token_usage_idempotency_key("message-1", 1) + attempt_two = delivery.token_usage_idempotency_key("message-1", 2) + require(gen_five != gen_six, "notification keys must vary by generation") + require(attempt_one != attempt_two, "token usage keys must vary by attempt") + + +def test_format_delivery_timestamp_normalizes_naive_and_aware_datetimes(): + naive = datetime(2026, 5, 4, 15, 0, 1, 123456) + aware = datetime(2026, 5, 4, 11, 0, 1, 123456, tzinfo=timezone(timedelta(hours=-4))) + require(delivery.format_delivery_timestamp(naive) == "2026-05-04T15:00:01.123456Z", "naive timestamps should be treated as UTC") + require(delivery.format_delivery_timestamp(aware) == "2026-05-04T15:00:01.123456Z", "aware timestamps should normalize to UTC") + require(delivery.format_delivery_timestamp("not-datetime") is None, "non-datetime values should not format") + + +def test_parse_delivery_timestamp_round_trips_and_normalizes(): + original = datetime(2026, 5, 4, 11, 0, 1, tzinfo=timezone(timedelta(hours=-4))) + formatted = delivery.format_delivery_timestamp(original) + parsed = delivery.parse_delivery_timestamp(formatted) + require(parsed == datetime(2026, 5, 4, 15, 0, 1, tzinfo=timezone.utc), "format/parse should round trip through UTC") + naive = delivery.parse_delivery_timestamp("2026-05-04T15:00:01") + require(naive == datetime(2026, 5, 4, 15, 0, 1, tzinfo=timezone.utc), "naive parsed timestamps should become UTC") + + +def test_parse_delivery_timestamp_returns_none_for_junk_inputs(): + for value in ("", "not-a-date", "1" * 65, object(), None): + parsed = delivery.parse_delivery_timestamp(value) + require(parsed is None, f"junk timestamp should parse as None: {value!r}") + + +def test_delivery_label_includes_quoted_name_and_asked_when(): + label = delivery.delivery_label("Daily digest", "Sun Jan 4, 2026, 9:30 PM EST") + without_time = delivery.delivery_label("Daily digest", "") + require(label == "Results from `Daily digest` ยท you asked on Sun Jan 4, 2026, 9:30 PM EST", "delivery label format changed") + require(without_time == "Results from `Daily digest`", "empty asked time should be omitted from delivery label") + + +def test_format_workflow_run_time_pins_asked_time_in_new_york(): + rendered = format_workflow_run_time("2026-01-05T02:30:00Z", "America/New_York") + require(rendered == "Sun Jan 4, 2026, 9:30 PM EST", "asked time format for America/New_York changed") + + +def test_format_workflow_run_time_falls_back_to_utc_for_bad_or_missing_zone(): + bad_zone = format_workflow_run_time("2026-01-05T02:30:00Z", "Not/A_Real_Zone") + missing_zone = format_workflow_run_time("2026-01-05T02:30:00Z", None) + expected = "Mon Jan 5, 2026, 2:30 AM UTC" + require(bad_zone == expected, "invalid time zone should fall back to UTC") + require(missing_zone == expected, "missing time zone should fall back to UTC") + + +def test_clean_catalog_text_drops_noise_collapses_space_and_caps_length(): + cleaned = delivery.clean_catalog_text(" Alpha\u200b\n\tBeta ", 80) + capped = delivery.clean_catalog_text("abcdefghij", 6) + require(cleaned == "Alpha Beta", "catalog text should remove controls and collapse whitespace") + require(capped == "abcdeโ€ฆ", "catalog text should cap with an ellipsis") + require(delivery.clean_catalog_text(42, 80) == "", "non-string catalog text should become empty") + + +def test_quoted_workflow_name_neutralizes_backticks_noise_and_empty_names(): + quoted = delivery.quoted_workflow_name("Bad`Name\u200b\nNext") + empty = delivery.quoted_workflow_name("```\u200b") + require(quoted == "`Bad'Name Next`", "quoted workflow name should neutralize markdown/control characters") + require(empty is None, "names with nothing but backticks and noise should be omitted") + + +def test_quoted_workflow_name_caps_to_name_max_length(): + quoted = delivery.quoted_workflow_name("A" * 90) + inner = quoted.strip("`") + require(len(inner) == delivery.NAME_MAX_LENGTH, "quoted workflow name should cap inner text at NAME_MAX_LENGTH") + require(inner.endswith("โ€ฆ"), "capped workflow names should end with an ellipsis") + + +def test_display_workflow_name_falls_back_to_default_workflow(): + require(delivery.display_workflow_name("Daily digest") == "`Daily digest`", "display name should quote real names") + require(delivery.display_workflow_name("```") == "`Workflow`", "display name should fall back for empty sanitized names") + + +def test_notice_workflow_name_uses_quotes_and_default_text(): + named = delivery.notice_workflow_name('Daily "digest"') + fallback = delivery.notice_workflow_name("\u200b") + require(named == '"Daily \'digest\'"', "notice workflow name should quote and neutralize double quotes") + require(fallback == "your workflow", "notice workflow name should fall back to your workflow") + + +def test_notice_title_pins_every_supported_kind(): + expected = { + delivery.KIND_RESULT: 'Results from "Daily digest" are ready', + delivery.KIND_ANALYSIS: 'Results from "Daily digest" are ready', + delivery.KIND_STATUS: 'Results from "Daily digest" are ready', + delivery.KIND_CONTENT_BLOCKED: 'Results from "Daily digest" are ready', + delivery.KIND_FAILED: '"Daily digest" didn\'t finish', + delivery.KIND_CANCELLED: '"Daily digest" was cancelled', + delivery.KIND_SKIPPED: '"Daily digest" didn\'t run', + delivery.KIND_EXPIRED: '"Daily digest" didn\'t finish in time to post to chat', + } + for kind, title in expected.items(): + actual = delivery.notice_title(kind, "Daily digest") + require(actual == title, f"notice title changed for {kind}") + fallback = delivery.notice_title(delivery.KIND_RESULT, "\u200b") + require(fallback == "Results from your workflow are ready", "default notice title sentence casing changed") + + +def test_delivered_notice_preview_is_pinned(): + preview = delivery.delivered_notice_preview("Daily digest") + fallback = delivery.delivered_notice_preview("\u200b") + require(preview == 'Results from "Daily digest" are in your chat', "delivered preview changed") + require(fallback == "Results from your workflow are in your chat", "default delivered preview changed") + + +def test_assemble_delivery_content_joins_non_empty_parts_with_blank_lines(): + content = delivery.assemble_delivery_content("Label", " Body\n", "Disclosure") + body_only = delivery.assemble_delivery_content("", " Body ", "") + require(content == "Label\n\nBody\n\nDisclosure", "delivery content layout changed") + require(body_only == "Body", "empty label/disclosure should be omitted") + + +def test_failure_reason_text_pins_all_known_codes_and_unknown_default(): + expected = { + "failed": "the run stopped before it finished", + "invalid": "the workflow couldn't run as defined", + "incomplete": "some of its steps didn't finish", + "skipped": "no new or changed files were detected", + delivery.REASON_DEADLINE_EXCEEDED: "it reached its time limit", + "execution_budget_exceeded": "it reached its execution limit", + "repeat_iteration_limit": "it reached its repeat limit", + "m365_authorization": "Microsoft 365 access needs to be reconnected", + "authorization": "your workflow access must be restored", + } + require(set(expected) == set(delivery.FAILURE_REASON_CODES), "failure reason code inventory changed") + for code, text in expected.items(): + actual = delivery.failure_reason_text(code) + require(actual == text, f"failure reason text changed for {code}") + require(delivery.failure_reason_text("new-code") == expected["failed"], "unknown reason codes should use failed text") + + +def test_delivery_note_text_pins_supported_kinds_and_reason_codes(): + expected = { + delivery.KIND_FAILED: "`Daily digest` failed: it reached its time limit.", + delivery.KIND_CANCELLED: "`Daily digest` was cancelled.", + delivery.KIND_STATUS: "`Daily digest` finished. Open the run to see its results.", + delivery.KIND_CONTENT_BLOCKED: "`Daily digest` finished, but its results can't be shown here. Open the run to see them.", + delivery.KIND_ANALYSIS: "`Daily digest` finished with a saved analysis. Ask a follow-up question about it here.", + delivery.KIND_SKIPPED: "`Daily digest` didn't run: no new or changed files were detected.", + } + for kind, text in expected.items(): + reason = delivery.REASON_DEADLINE_EXCEEDED if kind == delivery.KIND_FAILED else None + actual = delivery.delivery_note_text(kind, "Daily digest", reason_code=reason) + require(actual == text, f"delivery note text changed for {kind}") + require(bool(actual), f"delivery note text should not be empty for {kind}") + + +def test_delivery_note_text_rejects_unsupported_expired_kind(): + with pytest.raises(ValueError, match="unsupported delivery note kind"): + delivery.delivery_note_text(delivery.KIND_EXPIRED, "Daily digest") + + +def test_build_delivery_metadata_shape_and_values_are_pinned(): + metadata = delivery.build_delivery_metadata( + kind=delivery.KIND_RESULT, + workflow_id=WORKFLOW_ID, + run_id=RUN_ID, + generation=5, + run_status="completed", + orchestration_run_id="orchestration-run-1", + step_id=STEP_ID, + requested_at=REQUESTED_AT, + ) + expected = { + "version": delivery.CHAT_DELIVERY_VERSION, + "kind": delivery.KIND_RESULT, + "workflow_id": WORKFLOW_ID, + "workflow_scope": delivery.WORKFLOW_SCOPE, + "run_id": RUN_ID, + "generation": 5, + "run_status": "completed", + "orchestration_run_id": "orchestration-run-1", + "step_id": STEP_ID, + "requested_at": REQUESTED_AT, + } + require(metadata == expected, "delivery metadata snapshot shape changed") + + +def test_notice_metadata_shape_and_values_are_pinned(): + metadata = delivery.notice_metadata(WORKFLOW_ID, RUN_ID, delivery.STATUS_UNDELIVERABLE) + require(metadata == { + "workflow_id": WORKFLOW_ID, + "run_id": RUN_ID, + "workflow_scope": delivery.WORKFLOW_SCOPE, + "delivery_status": delivery.STATUS_UNDELIVERABLE, + }, "notice metadata shape changed") + + +def test_workflow_run_notice_link_url_encodes_odd_ids(): + link = delivery.workflow_run_notice_link("workflow id/&?", "run id/&?") + require( + link == "/workflow-activity?workflowId=workflow+id%2F%26%3F&runId=run+id%2F%26%3F&scope=personal", + "workflow run notice link encoding changed", + ) + + +def test_workflow_delivery_refusal_payloads_are_pinned(): + retry_payload, retry_status = delivery.workflow_delivery_refusal_payload(delivery.DELIVERY_RETRY_UNSUPPORTED) + edit_payload, edit_status = delivery.workflow_delivery_refusal_payload(delivery.DELIVERY_EDIT_UNSUPPORTED) + require(retry_status == 400 and edit_status == 400, "refusal status should stay 400") + require(retry_payload == { + "error": "A workflow run posted this message, so it can't be retried here. To run the workflow again, open the run in Workflows.", + "code": delivery.DELIVERY_RETRY_UNSUPPORTED, + }, "retry refusal payload changed") + require(edit_payload == { + "error": "A workflow run posted this message, so it can't be edited. Ask a new question instead.", + "code": delivery.DELIVERY_EDIT_UNSUPPORTED, + }, "edit refusal payload changed") + payload, status = delivery.workflow_delivery_refusal_payload("bogus") + require(status == 400, "unknown refusal status should stay 400") + require(payload["code"] == delivery.DELIVERY_RETRY_UNSUPPORTED, "unknown refusal code should fall back to retry") + + +def test_next_backoff_seconds_follows_table_and_clamps_edges(): + attempts = [-10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 99, "2"] + expected = [30, 30, 30, 60, 120, 240, 480, 900, 900, 900, 900, 30] + actual = [delivery.next_backoff_seconds(value) for value in attempts] + require(actual == expected, "backoff table or clamping changed") + + +def test_normalize_requester_roles_dedupes_preserves_order_and_drops_bad_values(): + roles = ["User", "", None, "Admin", "User", "x" * 65, "Approver"] + normalized = delivery.normalize_requester_roles(roles) + require(normalized == ["User", "Admin", "Approver"], "role normalization should drop invalid roles and dedupe in order") + require(delivery.normalize_requester_roles("User") == [], "non-list roles should normalize to empty") + many_roles = [f"role-{index}" for index in range(delivery.MAX_ROLES + 3)] + capped = delivery.normalize_requester_roles(many_roles) + require(len(capped) == delivery.MAX_ROLES, "role normalization should cap the number of roles") + require(capped[-1] == f"role-{delivery.MAX_ROLES - 1}", "role normalization should preserve first MAX_ROLES entries") + + +def test_normalize_model_selection_keeps_valid_model_reasoning_and_groups(): + seeds = { + "model": { + "model_deployment": " gpt-4o ", + "model_id": "model-1", + "model_endpoint_id": "endpoint-1", + "model_provider": "aoai", + }, + "reasoning_effort": " high ", + "active_group_ids": [" group-1 ", "group-2", "group-1", ""], + } + normalized = delivery.normalize_model_selection(seeds) + require(normalized == { + "model": { + "model_deployment": "gpt-4o", + "model_id": "model-1", + "model_endpoint_id": "endpoint-1", + "model_provider": "aoai", + }, + "reasoning_effort": "high", + "active_group_ids": ["group-1", "group-2"], + }, "model selection normalization changed for valid seeds") + + +def test_normalize_model_selection_drops_invalid_model_and_uses_fallback_groups(): + seeds = {"model": {"model_deployment": 123}, "reasoning_effort": "x" * 40} + normalized = delivery.normalize_model_selection(seeds, active_group_ids=["g1", "g1", "g2"]) + require(normalized == {"model": {}, "reasoning_effort": "", "active_group_ids": ["g1", "g2"]}, "invalid model fields should clear model and use fallback groups") + groups = [f"g-{index}" for index in range(delivery.MAX_GROUPS + 3)] + capped = delivery.normalize_model_selection({}, active_group_ids=groups) + require(capped["active_group_ids"] == groups[:delivery.MAX_GROUPS], "active groups should cap at MAX_GROUPS") + + +def test_normalize_time_zone_keeps_valid_names_and_rejects_junk(): + require(delivery.normalize_time_zone("America/New_York") == "America/New_York", "valid IANA-looking zone should be kept") + for zone in (None, "", "Europe/Paris;drop", "A" * 65, "America New_York", "Etc/UTC