diff --git a/application/single_app/config.py b/application/single_app/config.py index ef6a7d47c..9eba2c3d2 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.302" +VERSION = "0.261.303" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_orchestration_artifacts.py b/application/single_app/functions_orchestration_artifacts.py index 9a27973e0..9434f3da2 100644 --- a/application/single_app/functions_orchestration_artifacts.py +++ b/application/single_app/functions_orchestration_artifacts.py @@ -4,10 +4,15 @@ The application owner registers an actor-scoped service factory at bootstrap. This module never discovers clients, settings, credentials, or source paths. The output row, not file-message metadata, authorizes visibility. +A file's publication is authorized by the rendering service running its attempt on the +same task, so a signed-in execution checks the file's sources with that sign-in instead of +a fresh service that has none (0.261.303). """ import re import uuid +from contextlib import contextmanager +from contextvars import ContextVar from copy import deepcopy from azure.core import MatchConditions @@ -27,12 +32,15 @@ parse_time, safe_identity, ) -from functions_orchestration_result_contracts import ProducerIdentity, ResultContractError, canonical_digest +from functions_orchestration_result_contracts import ( + EXTERNAL_SESSION_UNAVAILABLE_REASON, ProducerIdentity, ResultContractError, canonical_digest, +) ORCHESTRATION_ARTIFACT_KIND = "orchestration_retained_output" ORCHESTRATION_ARTIFACT_VERSION = 1 ORCHESTRATION_ARTIFACT_KEY_PREFIX = "orchestration-output:v1:" +OUTPUT_SIGN_IN_REQUIRED = "output_sign_in_required" _DIGEST = re.compile(r"[a-f0-9]{64}\Z") _OUTPUT_ID = re.compile(r"orender_[a-f0-9]{64}\Z") _SOURCE_FIELDS = frozenset({ @@ -40,6 +48,31 @@ "source_digest", "spec_digest", }) _service_factory = None +# The rendering service running a file attempt on this task. Never shared across threads. +_attempt_service = ContextVar("orchestration_render_attempt_service", default=None) + + +@contextmanager +def render_attempt_scope(service): + """Authorize this task's file publications with the service that is rendering them.""" + token = _attempt_service.set(service) + try: + yield + finally: + _attempt_service.reset(token) + + +def signed_in_session_required(error): + """Whether a refusal came only from having no signed-in session to check current access.""" + seen = set() + current = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + for reason in (getattr(current, "code", None), getattr(current, "authority_reason", None)): + if type(reason) is str and reason in (EXTERNAL_SESSION_UNAVAILABLE_REASON, OUTPUT_SIGN_IN_REQUIRED): + return True + current = current.__cause__ + return False def _external_authority_failure(error, *, include_configuration): @@ -277,7 +310,16 @@ def _conversation_service(user_id, conversation_id): def _bound_service(user_id, value): binding = validate_orchestration_artifact_binding(value) producer = binding["producer"] - if user_id != producer["user_id"] or _service_factory is None: + if user_id != producer["user_id"]: + raise OutputUnavailableError("output_access_denied") + attempt_service = _attempt_service.get() + if ( + attempt_service is not None + and attempt_service.store.user_id == user_id + and attempt_service.store.conversation_id == producer["conversation_id"] + ): + return attempt_service, binding + if _service_factory is None: raise OutputUnavailableError("output_access_denied") return _conversation_service(user_id, producer["conversation_id"]), binding @@ -311,7 +353,9 @@ def load_orchestration_artifact_binding( raise OutputStorageError() from exc except (PermissionError, LookupError, ValueError, TypeError, RuntimeError) as exc: _raise_typed_artifact_failure(exc) - raise OutputUnavailableError("output_source_unavailable") from exc + raise OutputUnavailableError( + OUTPUT_SIGN_IN_REQUIRED if signed_in_session_required(exc) else "output_source_unavailable", + ) from exc except (AttributeError, OSError) as exc: _raise_typed_artifact_failure(exc) raise diff --git a/application/single_app/functions_orchestration_checkpoints.py b/application/single_app/functions_orchestration_checkpoints.py index 079dcd0e3..11c5b6ce0 100644 --- a/application/single_app/functions_orchestration_checkpoints.py +++ b/application/single_app/functions_orchestration_checkpoints.py @@ -1,7 +1,8 @@ # functions_orchestration_checkpoints.py """Private, immutable step-boundary checkpoints in the run-steps partition. -Version: 0.261.139 +Version: 0.261.303 +Settings fingerprints ignore storage metadata and runtime state in: 0.261.303 The lifecycle row fences every batch, including uncommitted chunks. It survives cleanup, so an old worker cannot recreate payloads after conversation deletion. Checkpoints record Gather / Reason / Render state only. A checkpoint written by the @@ -26,6 +27,7 @@ ProducerIdentity, ResultContractError, ResultRef, TaskResult, digest as validate_result_digest, ) from functions_orchestration_result_runtime import encode_step_result +from functions_settings_runtime_state import configuration_settings CHECKPOINT_VERSION = 1 @@ -100,8 +102,13 @@ def orchestration_answer_message_id(run_id): def _execution_settings_fingerprint(settings): - """The settings a saved attempt's work was approved and executed under.""" - return fingerprint(settings) + """The settings a saved attempt's work was approved and executed under. + + Storage metadata and the runtime state background tasks write are not policy. They + change on every save, so including them failed waiting runs and retries with + ``recovery_changed`` whenever anything saved settings. + """ + return fingerprint(configuration_settings(settings) if isinstance(settings, dict) else settings) def _manifest_address(step_id, *, waiting=False, input_only=False): @@ -252,6 +259,21 @@ def step_input_fingerprint(step, context, binding, *, settings=None): }) +def _ended_attempt_waits(tasks, run_id): + """Steps whose saved result is a wait that an earlier attempt left unfinished. + + A checkpoint saves every result its attempt held, including the pending result of a step + that was still waiting. A new attempt runs that step again, so the old wait is not restored: + it has no outputs that anything could have read, and only its own attempt could finish it. + """ + if type(run_id) is not str: + return set() + return { + name for name, task in tasks.items() + if task.status == 'pending' and not task.outputs and task.producer.run_id != run_id + } + + def restore_context(context, payload): state = payload.get('state') allowed = set(STATE_FIELDS) | set(OPTIONAL_STATE_FIELDS) | set(DEPENDENCY_STATE_FIELDS) @@ -274,6 +296,7 @@ def restore_context(context, payload): raise CheckpointError('checkpoint_invalid') setattr(context, key, deepcopy(value)) try: + ended_waits = set() for field, descriptor_type in (('task_results', TaskResult), ('result_aliases', ResultRef)): values = state.get(field) if type(values) is not dict: @@ -282,8 +305,12 @@ def restore_context(context, payload): current = getattr(context, field) if field == 'task_results' and any(name != value.producer.step_id for name, value in restored.items()): raise CheckpointError('checkpoint_invalid') + if field == 'task_results': + ended_waits = _ended_attempt_waits(restored, getattr(context, 'run_id', None)) for name, value in restored.items(): - if field == 'task_results' and name in getattr(context, '_failed_result_step_ids', ()): + if field == 'task_results' and ( + name in getattr(context, '_failed_result_step_ids', ()) or name in ended_waits + ): continue previous = current.get(name) if previous is not None and previous != value: @@ -308,6 +335,7 @@ def restore_context(context, payload): (task is not None and task.status in ('complete', 'partial')) or step_id in getattr(context, '_completed_result_step_ids', ()) or step_id in getattr(context, '_failed_result_step_ids', ()) + or step_id in ended_waits ): continue if step_id in context.pending_results and context.pending_results[step_id] != wait: diff --git a/application/single_app/functions_orchestration_continuation.py b/application/single_app/functions_orchestration_continuation.py index c39b1584b..76c41ae6a 100644 --- a/application/single_app/functions_orchestration_continuation.py +++ b/application/single_app/functions_orchestration_continuation.py @@ -49,7 +49,7 @@ lease_fields, reconcile_checkpoints, ) -from functions_orchestration_rendering import raise_output_read_infrastructure_failure +from functions_orchestration_rendering import raise_output_read_infrastructure_failure, render_failure_code from functions_orchestration_result_contracts import TaskResult from functions_orchestration_result_runtime import decode_step_result, validate_task_outputs from functions_orchestration_schema import ( @@ -764,7 +764,9 @@ def reconcile_run_outputs(record, *, services, lease): pending[step_id] = deepcopy(row.get("wait") or wait) else: pending.pop(step_id, None) - failure = build_failure("step_failed", step_id=step_id, capability_id="render_file") if status == "failed" else None + failure = build_failure( + render_failure_code(fact.get("error_code")), step_id=step_id, capability_id="render_file", + ) if status == "failed" else None failures = [value for value in failures if value.get("step_id") != step_id] if failure is not None: failures.append(failure) diff --git a/application/single_app/functions_orchestration_output_store.py b/application/single_app/functions_orchestration_output_store.py index 20b2c8dc1..ae56f6c67 100644 --- a/application/single_app/functions_orchestration_output_store.py +++ b/application/single_app/functions_orchestration_output_store.py @@ -49,11 +49,31 @@ "output_artifact_missing": "This file is unavailable because its committed artifact is missing.", "output_artifact_mismatch": "This file is unavailable because its artifact binding could not be verified.", "output_intent_invalid": "This file is unavailable because its artifact binding could not be verified.", + "output_sign_in_required": ( + "This file is unavailable here because the results it uses can only be checked with your sign-in." + ), } -# A file a time budget stopped says so instead of the generic failed or cancelled text. +# A failed file says why it could not be created instead of the generic failed text. A file a +# time budget stopped names that budget. OUTPUT_FAILURE_MESSAGES = { "output_step_time_limit": "This file was stopped because its step reached the time limit.", "output_deadline_exceeded": "This file was stopped because the run reached its time limit.", + "output_sign_in_required": ( + "This file could not be created because the results it uses can only be checked with your " + "sign-in, which wasn't available where the file was prepared. Retry from this chat to create it." + ), + "output_access_denied": ( + "This file could not be created because access to its conversation or to the results it uses " + "could not be confirmed." + ), + "output_source_unavailable": ( + "This file could not be created because a result it uses is missing or no longer readable." + ), + "output_source_changed": ( + "This file could not be created because a result it uses no longer matches the retained version." + ), + "output_screening_hold": "This file could not be created while a source it uses is under review.", + "output_capability_disabled": "This file could not be created under the current capability settings.", } _ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,199}\Z") _OUTPUT_ID = re.compile(r"orender_[a-f0-9]{64}\Z") @@ -169,6 +189,7 @@ def public_output(record, *, unavailable_code=None, withhold_details=False): if withhold_details or not available or record.get("error_code") in { "output_access_denied", "output_screening_hold", "output_deleted", "output_conversation_unavailable", "output_source_unavailable", "output_superseded", + "output_sign_in_required", }: descriptor = {} projected = { diff --git a/application/single_app/functions_orchestration_recovery.py b/application/single_app/functions_orchestration_recovery.py index f47e7cefd..d5204555b 100644 --- a/application/single_app/functions_orchestration_recovery.py +++ b/application/single_app/functions_orchestration_recovery.py @@ -1,7 +1,7 @@ # functions_orchestration_recovery.py """Execution leases and explicitly requested, checkpoint-only retry attempts. -Version: 0.261.293 +Version: 0.261.303 Retry publication is one transactional parent CAS + child create. It never replans, invokes an adapter, or changes plan-revision lineage. Terminal publication preserves an administrator's reply retraction; its probe @@ -11,6 +11,8 @@ file admissions, because preparing the retry supersedes the parent's files. A retry runs again a step that finished without the chart or diagram the plan asked of it, and asks for confirmation when that step can have external effects (0.261.293). +A wait left unfinished by an attempt that failed or was stopped no longer blocks a retry; +the retry runs the waiting step again (0.261.303). A retry that could only resend requests a service declined is not offered. A run from the removed legacy contract is never retried, resumed or continued; only conversation deletion still reads it, to remove its saved data. @@ -480,7 +482,9 @@ def recovery_projection(record): reason, message = 'legacy_no_checkpoints', 'This run has no durable checkpoints and cannot resume. Create a new plan.' elif _live(record): reason, message = 'execution_live', 'This attempt is still running. Wait for it to finish or stop it.' - elif any(step.get('status') == 'waiting' for step in steps): + elif record.get('status') not in _TERMINAL and any(step.get('status') == 'waiting' for step in steps): + # A wait only blocks retry while its attempt can still continue. Once the attempt has + # ended, the waiting step and the steps after it run again in the new attempt. reason, message = 'result_not_ready', build_failure('result_not_ready')['message'] elif any(step.get('status') == 'running' for step in steps): reason, message = 'result_commit_unconfirmed', build_failure('result_commit_unconfirmed')['message'] @@ -1130,6 +1134,10 @@ def validate_resume(record, context, settings, authorize, *, source_run_id=None, saved = source_steps.get(step['step_id']) or {} if saved.get('status') == 'waiting': if not allow_waiting: + if source.get('status') in _TERMINAL: + # The attempt that waited has ended, so nothing will finish this wait. + # A new attempt runs the step again instead of reusing it. + continue raise CheckpointError('result_not_ready') source_store = checkpoint_store(source, authorize) if not source_store.has_manifest(step['step_id'], waiting=True): diff --git a/application/single_app/functions_orchestration_rendering.py b/application/single_app/functions_orchestration_rendering.py index e6f3c844f..3a5df0617 100644 --- a/application/single_app/functions_orchestration_rendering.py +++ b/application/single_app/functions_orchestration_rendering.py @@ -4,6 +4,8 @@ No producer, composition model, executor, route, or configuration owner is imported. Runtime owners supply current capability/admission checks and private transport. Approved deadlines and retry admissions survive worker replacement. +A file whose sources could only be checked with a signed-in session that wasn't +available fails as ``output_sign_in_required`` and says so (0.261.303). """ import hashlib @@ -38,6 +40,7 @@ from functions_generated_export_registry import ASSEMBLED_DOCUMENT_PROFILE, resolve_generated_file_export_format from functions_generated_file_exports import build_generated_file_export from functions_orchestration_artifacts import ( + OUTPUT_SIGN_IN_REQUIRED, OrchestrationArtifactTransport, assert_artifact_matches, binding_for_intent, @@ -45,10 +48,13 @@ external_authority_failure, orchestration_artifact_file_extensions, output_storage_failure, + render_attempt_scope, + signed_in_session_required, validate_orchestration_artifact_binding, ) from functions_orchestration_export_sources import open_orchestration_export_source from functions_orchestration_output_store import ( + OUTPUT_FAILURE_MESSAGES, OUTPUT_UNAVAILABLE_MESSAGES, OrchestrationOutputStore, OutputClaim, @@ -164,6 +170,8 @@ def _source_visibility_code(error): return "output_source_changed" if error.code in {"result_producer_unavailable", "result_attempt_mismatch", "result_attempt_stopped"}: return "output_source_unavailable" + if signed_in_session_required(error): + return OUTPUT_SIGN_IN_REQUIRED return "output_access_denied" if isinstance(error, PermissionError): return "output_access_denied" @@ -252,6 +260,9 @@ def output_failure(exc): return authority[1], authority[2] if output_storage_failure(exc) is not None: return "output_storage_unavailable", True + if signed_in_session_required(exc): + # Automatic and file-only retries render in the background, which has no sign-in. + return OUTPUT_SIGN_IN_REQUIRED, False if isinstance(exc, DocumentHeldError): return "output_screening_hold", False if isinstance(exc, SourceAuthorityUnavailableError): @@ -1019,7 +1030,10 @@ def render_attempt(self, output_id, *, claim=None, worker_id=None, stop=None, ob attempt = _RenderAttempt(self, stop=stop) outcome = failure = None try: - outcome = self._render_attempt(output_id, claim, worker_id, attempt) + # Publishing the file rechecks its sources with this service's identity, not a + # fresh service's, so a signed-in execution can publish what it rendered. + with render_attempt_scope(self): + outcome = self._render_attempt(output_id, claim, worker_id, attempt) return outcome except Exception as exc: failure = exc @@ -1273,6 +1287,11 @@ def check(): yield stream +def render_failure_code(error_code): + """The step failure a failed file reports when it carries a reason of its own.""" + return "file_sign_in_required" if error_code == OUTPUT_SIGN_IN_REQUIRED else "step_failed" + + def _render_step_result(output, record, *, build_step_result, build_failure): if output["state"] == "completed": result = build_step_result( @@ -1288,7 +1307,10 @@ def _render_step_result(output, record, *, build_step_result, build_failure): result = build_step_result(status="cancelled", summary=output["message"]) else: # A file its step's time budget stopped reports that budget, with its admin hint. - failure_code = "step_timeout" if output.get("error_code") == OutputStepTimeLimitError.code else "step_failed" + failure_code = ( + "step_timeout" if output.get("error_code") == OutputStepTimeLimitError.code + else render_failure_code(output.get("error_code")) + ) result = build_step_result( status="failed", summary=output["message"], failure=build_failure(failure_code), error=output["message"], @@ -1496,9 +1518,13 @@ def execute_render_file( result["outputs"] = [] if uncertain_authority or output["state"] == "completed" else [output] result["output_error"] = {"code": code, "retryable": retryable} return result + message = ( + OUTPUT_FAILURE_MESSAGES[code] if code == OUTPUT_SIGN_IN_REQUIRED + else "This file could not be created." + ) result = build_step_result( - status="failed", summary="This file could not be created.", - failure=build_failure("step_failed"), error="This file could not be created.", + status="failed", summary=message, + failure=build_failure(render_failure_code(code)), error=message, ) result["output_error"] = {"code": code, "retryable": retryable} return result diff --git a/application/single_app/functions_orchestration_schema.py b/application/single_app/functions_orchestration_schema.py index b2024fd71..66e235b9a 100644 --- a/application/single_app/functions_orchestration_schema.py +++ b/application/single_app/functions_orchestration_schema.py @@ -1865,6 +1865,11 @@ def summarize_plan(plan): ), 'document_merge_failed': "The selected files couldn't be merged into one file.", 'step_failed': 'This operation could not complete.', + 'file_sign_in_required': ( + 'A file could not be created because the results it uses can only be checked with your ' + "sign-in, which wasn't available where the file was prepared. Select Retry from failed step " + 'to create it again.' + ), 'message_not_saved': 'The explanation could not be saved. Reload this run to check its durable status.', LEGACY_PLAN_CODE: LEGACY_PLAN_MESSAGE, } diff --git a/application/single_app/functions_settings_runtime_state.py b/application/single_app/functions_settings_runtime_state.py new file mode 100644 index 000000000..46a5b8213 --- /dev/null +++ b/application/single_app/functions_settings_runtime_state.py @@ -0,0 +1,67 @@ +# functions_settings_runtime_state.py +"""Settings keys that record the application's own runtime state, not configuration. + +Version: 0.261.303 +Implemented in: 0.261.303 + +The settings document also holds values the application writes for itself while it +runs: storage metadata that changes on every save, background monitor readings and +schedule bookkeeping. An administrator never approved any of them as policy, so a +fingerprint of "the settings this work ran under" has to leave them out. Including them +made any background save look like a policy change to saved orchestration progress. + +Configuration an administrator saves stays in the fingerprint, so a real policy change +still fails closed. This module imports only owners that never load configuration, so +low-level modules such as orchestration checkpoints can use it. +""" + +from app_settings_store import COSMOS_METADATA_FIELDS, SETTINGS_REVISION_FIELD +from functions_logging_timers import LOGGING_TIMERS + + +# Written by the settings store itself on every save. +SETTINGS_STORAGE_KEYS = frozenset({*COSMOS_METADATA_FIELDS, SETTINGS_REVISION_FIELD, "id"}) + +# Written by background tasks and request side effects as the application runs. +SETTINGS_RUNTIME_STATE_KEYS = frozenset({ + # Semantic search quota warnings, recorded and cleared by searches. + "service_health", + # Control Center auto-refresh bookkeeping. + "control_center_last_refresh", + # Retention policy run bookkeeping. + "retention_policy_last_run", + "retention_policy_next_run", + # The cached application release check. + "last_update_check_time", + "last_update_check_attempt_time", + "last_update_check_failed", + "latest_version_available", + "update_available", + # Log switches and their timers, which the background checker turns off. + *(key for timer in LOGGING_TIMERS.values() for key in timer.values()), +}) + +SETTINGS_RUNTIME_STATE_PREFIXES = ( + # Cosmos DB throughput monitoring: readings, scale history and the per-container + # policies whose scale timestamps the monitor updates. + "cosmos_throughput_", + # The Control Center auto-refresh schedule the refresh task seeds and advances. + "control_center_auto_refresh_", +) + + +def is_settings_runtime_state_key(key): + """Whether a settings key holds storage metadata or runtime state rather than configuration.""" + return type(key) is str and ( + key in SETTINGS_STORAGE_KEYS + or key in SETTINGS_RUNTIME_STATE_KEYS + or key.startswith(SETTINGS_RUNTIME_STATE_PREFIXES) + ) + + +def configuration_settings(settings): + """A shallow copy of ``settings`` without storage metadata or runtime state.""" + return { + key: value for key, value in (settings or {}).items() + if not is_settings_runtime_state_key(key) + } diff --git a/application/v2_ui/src/components/chat/OrchestrationDeliverables.tsx b/application/v2_ui/src/components/chat/OrchestrationDeliverables.tsx index 62648c6b7..1e2d0957e 100644 --- a/application/v2_ui/src/components/chat/OrchestrationDeliverables.tsx +++ b/application/v2_ui/src/components/chat/OrchestrationDeliverables.tsx @@ -5,7 +5,8 @@ // server checks each one against what it can actually produce. This section shows that list // before the steps: which step produces each deliverable, and, for one that cannot be produced // here, the server's reason in a warning tone. During and after a run the same rows follow their -// steps, so a file or image that did not arrive is never shown as delivered. After a run, the +// steps, so a file or image that did not arrive is never shown as delivered, and a run that failed +// or stopped never leaves a deliverable it did not finish reading as in progress. After a run, the // server's own check decides a chart or diagram, because its step can finish without the answer // showing it. // @@ -13,7 +14,9 @@ import { clsx } from 'clsx'; import { Ban, CheckCircle2, CircleAlert, Loader2, TriangleAlert, XCircle } from 'lucide-react'; -import type { OrchestrationDeliverableCheck, OrchestrationPlan, PlanEdits, StepStatus } from '../../lib/orchestration'; +import type { + OrchestrationDeliverableCheck, OrchestrationPlan, PlanEdits, PlanStatus, StepStatus, +} from '../../lib/orchestration'; import { deliverableRows, type DeliverableRow, type DeliverableState } from '../../lib/orchestrationPlan'; const STATE_TONE: Record = { @@ -78,6 +81,7 @@ export function OrchestrationDeliverables({ statusOf, edits, checks, + runStatus, compact = false, }: { plan: OrchestrationPlan; @@ -86,10 +90,12 @@ export function OrchestrationDeliverables({ edits?: PlanEdits; /** The server's checks, after a run, of the charts and diagrams the user asked for. */ checks?: OrchestrationDeliverableCheck[]; + /** The run's status, so a run that ended does not show its unfinished deliverables as in progress. */ + runStatus?: PlanStatus; /** The approval card lists only what the user asked for, without step names. */ compact?: boolean; }) { - const rows = deliverableRows(plan, statusOf ?? (() => undefined), edits, checks); + const rows = deliverableRows(plan, statusOf ?? (() => undefined), edits, checks, runStatus); const asked = rows.filter((row) => row.deliverable.requested === 'explicit'); const added = compact ? [] : rows.filter((row) => row.deliverable.requested !== 'explicit'); if (!asked.length && !added.length) return null; diff --git a/application/v2_ui/src/components/chat/OrchestrationRecoveryNotice.tsx b/application/v2_ui/src/components/chat/OrchestrationRecoveryNotice.tsx index aac5ef04d..59a2128d6 100644 --- a/application/v2_ui/src/components/chat/OrchestrationRecoveryNotice.tsx +++ b/application/v2_ui/src/components/chat/OrchestrationRecoveryNotice.tsx @@ -93,7 +93,16 @@ export function OrchestrationRecoveryNotice({ || Boolean(newer && newer !== runId) || Boolean(attempt.retry_of_run_id); const currentPlan = saved?.plan ?? plan; const recovery = attempt.recovery; - const fileOutputs = Boolean(attempt.outputs?.length); + const outputs = attempt.outputs ?? []; + const fileOutputs = outputs.length > 0; + // A file that failed for good cannot be retried on its own: a file retry runs in the + // background, where the sign-in some sources need is not available. A plan retry runs it + // again from this chat, as long as no other file is still being prepared. + const filesInProgress = outputs.some((output) => + output.state === 'waiting' || output.state === 'rendering' || output.state === 'retry_scheduled'); + const planRetryForFiles = fileOutputs && !filesInProgress + && outputs.some((output) => output.state === 'failed' && !output.can_retry); + const planRetryOffered = !fileOutputs || planRetryForFiles; // The server refuses runs from an earlier orchestration version: there is nothing to reload, // check or review, only its message to show. const legacy = Boolean(saved?.legacyPlan); @@ -113,7 +122,7 @@ export function OrchestrationRecoveryNotice({ if (!runId || !relevant) return null; const active = Object.values(inFlight).some((run) => run.conversationId === conversationId); - const retryAllowed = failed && !fileOutputs && !isOrchestrationRunPending(runState) && recovery?.eligible && recovery.expected_version + const retryAllowed = failed && planRetryOffered && !isOrchestrationRunPending(runState) && recovery?.eligible && recovery.expected_version && (!newer || newer === runId) && !saved?.transportUnknown; const retry = async (confirmedVersion?: string) => { const result = await retryOrchestrationRun(conversationId, runId, confirmedVersion); @@ -148,17 +157,19 @@ export function OrchestrationRecoveryNotice({

) : null} {failed && !saved?.transportUnknown && !legacy ? ( -

{fileOutputs ? 'Review each file separately. File retry controls do not repeat the plan or its producer tasks.' +

{fileOutputs && !planRetryForFiles ? 'Review each file separately. File retry controls do not repeat the plan or its producer tasks.' + : planRetryForFiles && recovery?.eligible && (!newer || newer === runId) + ? 'A file could not be created and cannot be retried on its own. Retry from failed step creates the plan\'s files again without repeating completed plan steps.' : recovery?.message || (newer && newer !== runId ? 'A newer execution attempt already exists. Review its saved result.' : recovery?.eligible ? 'Resume the saved plan without repeating completed plan steps.' : 'This historical attempt has no verified recovery checkpoint. It cannot be resumed; start a new plan deliberately if needed.')}

) : null} - {failed && !fileOutputs && recovery?.reused_step_ids.length ? ( + {failed && planRetryOffered && recovery?.reused_step_ids.length ? (

Reuse saved results: {recovery.reused_step_ids.map(titleFor).join(', ')}.

) : null} - {failed && !fileOutputs && recovery?.retry_step_ids.length ? ( + {failed && planRetryOffered && recovery?.retry_step_ids.length ? (

Execute on retry: {recovery.retry_step_ids.map(titleFor).join(', ')}.

) : null}
diff --git a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx index b1ce6351f..e48df6df5 100644 --- a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx +++ b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx @@ -49,6 +49,7 @@ import { stepReferenceIds, stepRoleLabel, stepRemovableDocumentIds, + stepUnfinishedByEndedRun, } from '../../lib/orchestrationPlan'; import type { CostClass, @@ -104,6 +105,11 @@ const STATUS_LABELS: Record = { cancelled: 'Cancelled', }; +// A step a failed or stopped run never finished. Nothing is working on it any more. +const UNFINISHED_STEP_LABEL = 'Not finished'; +const UNFINISHED_STEP_SUMMARY = 'The run ended before this step finished.'; +const UNFINISHED_STEP_TONE = 'bg-surface-3 text-text-3'; + /** A step argument key/value the run will use, minus the document fields shown as chips. */ function readableArguments(step: OrchestrationStep): Array<[string, string]> { // Spreadsheet merge and inspection settings are put in words, without restating defaults. @@ -329,7 +335,11 @@ export function OrchestrationRunView({ const willRun = step.enabled && !disabledStepIds.has(step.step_id); const rawStatus = stepRuntime[step.step_id]?.status ?? step.status; const status: StepStatus = willRun ? rawStatus : 'skipped'; - const summary = stepRuntime[step.step_id]?.summary ?? ''; + // A run that failed or stopped as a whole can leave a step pending, running or waiting. + const unfinished = willRun && stepUnfinishedByEndedRun(status, plan.status); + const statusText = unfinished ? UNFINISHED_STEP_LABEL + : status === 'running' && roleLabel ? stepRoleLabel(step, true) : STATUS_LABELS[status]; + const summary = unfinished ? UNFINISHED_STEP_SUMMARY : stepRuntime[step.step_id]?.summary ?? ''; const removed = new Set(edits.removed_document_ids[step.step_id] ?? []); const removable = new Set(stepRemovableDocumentIds(step)); const references = stepReferenceIds(step); @@ -376,12 +386,13 @@ export function OrchestrationRunView({ {step.estimated_cost} - {status === 'running' && roleLabel ? stepRoleLabel(step, true) : STATUS_LABELS[status]} + {statusText} {stepRuntime[step.step_id]?.reused ? ( @@ -633,7 +644,7 @@ export function OrchestrationRunView({ /> ) : ( - {STATUS_LABELS[status]} + {unfinished ? UNFINISHED_STEP_LABEL : STATUS_LABELS[status]} )}
@@ -678,6 +689,7 @@ export function OrchestrationRunView({ plan={plan} edits={edits} statusOf={(stepId) => stepRuntime[stepId]?.status} checks={savedRun?.deliverable_states} + runStatus={plan.status} /> {plan.final_response !== undefined ? ( diff --git a/application/v2_ui/src/lib/orchestrationPlan.ts b/application/v2_ui/src/lib/orchestrationPlan.ts index 0a2444a8f..5899dae49 100644 --- a/application/v2_ui/src/lib/orchestrationPlan.ts +++ b/application/v2_ui/src/lib/orchestrationPlan.ts @@ -266,6 +266,23 @@ const DELIVERABLE_STATE_LABELS: Record = { turned_off: 'Turned off', }; +/** Run statuses after which nothing in the run can still finish. */ +const ENDED_RUN_STATUSES: ReadonlySet = new Set(['failed', 'cancelled', 'superseded']); + +/** + * Whether a step's work was left unfinished by a run that has ended. + * + * A run can fail or stop as a whole while a step is still pending, running or waiting for a + * result. Nothing will finish that step any more, so it must not read as running or waiting. + */ +export function stepUnfinishedByEndedRun(status: StepStatus | undefined, runStatus?: PlanStatus): boolean { + return runStatus !== undefined && ENDED_RUN_STATUSES.has(runStatus) + && (status === undefined || status === 'pending' || status === 'running' || status === 'waiting'); +} + +/** Shown for a deliverable whose steps the run ended before finishing. */ +export const RUN_ENDED_DELIVERABLE_REASON = 'The run ended before this was finished.'; + /** * Each declared deliverable with the state its producing steps imply. * @@ -274,6 +291,8 @@ const DELIVERABLE_STATE_LABELS: Record = { * it off. The server's delivery notes remain the authority after a run; this is the preview. * `checks` are the server's own checks after a run of the charts and diagrams the user asked * for. They win over step status, because a step can finish without its chart being shown. + * `runStatus` is the run's own status: once the run has failed or stopped, a deliverable whose + * steps never finished is not delivered rather than in progress. * The implicit answer of plans that declared nothing is left out. */ export function deliverableRows( @@ -281,14 +300,17 @@ export function deliverableRows( statusOf: (stepId: string) => StepStatus | undefined, edits?: PlanEdits, checks?: OrchestrationDeliverableCheck[], + runStatus?: PlanStatus, ): DeliverableRow[] { const disabled = new Set(edits?.disabled_step_ids ?? []); const checked = new Map((checks ?? []).map((check) => [check.id, check])); + const runEnded = runStatus !== undefined && ENDED_RUN_STATUSES.has(runStatus); return (plan.deliverables ?? []).filter((deliverable) => !deliverable.implicit).map((deliverable) => { const producers = plan.steps.filter((step) => step.delivers?.includes(deliverable.id)); const enabled = producers.filter((step) => step.enabled && !disabled.has(step.step_id)); const check = checked.get(deliverable.id); let state: DeliverableState = 'planned'; + let endedReason = false; if (deliverable.status === 'unavailable') { state = 'unavailable'; } else if (producers.length > 0 && enabled.length === 0) { @@ -301,6 +323,9 @@ export function deliverableRows( state = 'delivered'; } else if (statuses.some((status) => status === 'failed' || status === 'skipped' || status === 'cancelled')) { state = 'not_delivered'; + } else if (runEnded && statuses.length > 0) { + state = 'not_delivered'; + endedReason = true; } else if (statuses.some((status) => status === 'running' || status === 'waiting' || status === 'partial')) { state = 'running'; } @@ -316,7 +341,8 @@ export function deliverableRows( steps: producers.map((step) => step.title || step.step_id), ...(deliverable.status === 'unavailable' ? { reason: deliverable.unavailable_message || 'This is not available here.' } - : state === 'not_delivered' && check?.message ? { reason: check.message } : {}), + : state === 'not_delivered' && check?.message ? { reason: check.message } + : endedReason ? { reason: RUN_ENDED_DELIVERABLE_REASON } : {}), }; }); } diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index 30d890169..693790752 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -739,6 +739,9 @@ names the check that failed in `sc_reason`. It shares `sc_run_id_hash` and | A question is asked that was already answered | The earlier answer may be outside retained message history and the activity ledger. | Check the history window and ledger limits; the ledger alone does not contain the full earlier answer. | | A pending plan reports changed conversation context | A referenced message or its visibility changed after planning. | Create a new plan using the current conversation. | | A web or research step says it continued in the background, where the user's sign-in is not available | The run continued without the user's browser session, for example after a restart, so their app roles couldn't be confirmed. | Ask the user to send the request again from the chat. The new run uses their signed-in session. | +| Before 0.261.303, a run waiting on a long tabular analysis replied "The request could not be completed. Saved step inputs changed." while the plan panel still showed the step **Waiting for results** and the answer **In progress**, with no **Retry from failed step** | A saved run's binding hashed the whole settings document, including storage metadata and runtime state that background tasks save. Any settings save, such as the Cosmos DB throughput autoscale recording a scale action, made the run's continuation see changed inputs. `[ORCHESTRATION_RUNS] Headless execution did not complete.` logs `sc_execution_code=recovery_changed` a few seconds after `[ASC] App settings updated and published successfully.` | Upgrade to 0.261.303 or later. Background saves no longer affect saved runs, and a run that ended while a step was waiting shows **Not finished** and offers **Retry from failed step**. Runs saved before the upgrade can't be retried; ask again. See the [runs failing while their work was still running fix]({{ '/explanation/fixes/ORCHESTRATION_SETTINGS_WRITE_RECOVERY_FIX/' | relative_url }}). | +| Before 0.261.303, a file built from web search, deep research, agent or action results always said "This file could not be created." and offered no retry | Publishing the file rechecked its sources through a new service built without the user's sign-in. `[ORCHESTRATION_EXTERNAL_SOURCES] A step's source was refused.` logs `sc_authority_reason=external_identity_session_unavailable`, then `[ORCHESTRATION_EXECUTOR] A file render attempt finished.` logs `sc_output_code=output_access_denied`. | Upgrade to 0.261.303 or later. See the [files from external results fix]({{ '/explanation/fixes/ORCHESTRATION_FILE_SIGN_IN_PUBLICATION_FIX/' | relative_url }}). | +| A file says its results can only be checked with the user's sign-in | The file was prepared in the background, for example after the run waited for a long computation, where the user's sign-in isn't available to check web, deep research, agent or action results. | Ask the user to select **Retry from failed step** in the chat. The retry runs with their sign-in and reuses the completed steps. | | Every web search, linked-page or deep research step says a required retained result is unavailable or changed | Before 0.261.209, orchestration checked each user's roles through Microsoft Graph, which needs `Directory.Read.All`, and refused research whenever query or linked-page planning was on. | Upgrade to 0.261.209 or later; no Graph permission or settings change is needed. If a step still fails, check `sc_authority_reason` on its failure event in [orchestration failure diagnostics](../reference/logging-tags.md#orchestration-failure-diagnostics). | | Every step completed, but the run is partially completed with "A required retained result is unavailable or changed" | Before 0.261.237, settings read from Cosmos DB instead of the shared Redis copy came back as an SDK type that the access recheck refused. That happens while any settings save is in progress, for example the Cosmos throughput autoscale saving its status, and on every read when Redis is off. The failure event logs `sc_authority_reason=result_external_context_unavailable`. | Upgrade to 0.261.237 or later. No setting change is needed. See the [settings document type fix]({{ '/explanation/fixes/ORCHESTRATION_SETTINGS_DOCUMENT_TYPE_FIX/' | relative_url }}). | | Plans never propose deep research or reading a link | The user does not hold the required app role, or the capability is disabled in its own settings group. | Confirm the user holds `DeepResearchUser` or `UrlAccessUser` where your deployment requires them, and that the capability is enabled outside this page. | diff --git a/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md b/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md index 4c98832e5..0d0ee21e4 100644 --- a/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md +++ b/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md @@ -1,10 +1,15 @@ # Orchestration Checkpoint Recovery -**Version: 0.261.232** +**Version: 0.261.303** Implemented in version: **0.261.105**, recorded in `application/single_app/config.py`. +Since **0.261.303**, a settings save made by a background task no longer +invalidates saved progress, and an attempt that ended while a step was waiting +can be retried. See the +[runs failing while their work was still running fix](../fixes/ORCHESTRATION_SETTINGS_WRITE_RECOVERY_FIX.md). + Since **0.261.232**, restored retained references take their access from their conversation and run: their source documents are not reread. See [Upload-only content screening](UPLOAD_ONLY_CONTENT_SCREENING.md). @@ -104,6 +109,19 @@ stays paused. Open it and use **Run prepared retry** to start that saved attempt **Check saved status** reconciles uncertain execution state, while **View current attempt** and **View previous attempt** navigate history without executing work. +### When an attempt ends while a step is waiting + +A step that hands work to a long computation, such as a native tabular analysis, +waits in the same attempt until its result is ready. If the attempt then fails or +is stopped as a whole, nothing will finish that wait. Since **0.261.303**: + +- The Run view shows the waiting step, and any step still pending or running, as + **Not finished**, and the deliverable it produces as **Not delivered**. +- **Retry from failed step** is offered. The new attempt runs the waiting step + again, with the steps after it, and reuses the steps that completed. +- A waiting attempt that can still continue doesn't offer a retry; its wait isn't + finished yet. + ## External effects and checkpoint boundaries A checkpoint represents one completed orchestration step. An agent step can @@ -129,10 +147,12 @@ retry review lists them as work that will execute: supersedes that attempt's files, so a retry never reuses a render step. The new attempt renders its files again from saved or newly prepared content, which calls no model, and the earlier attempt's files are then shown as superseded. A restart of the same attempt - still reuses its completed files. React V2 offers **Retry from failed step** only for an - attempt without files. An attempt with files is recovered one file at a time with - **Retry file**, which never repeats plan steps or withdraws an available file. A - whole-run retry prepared through the retry API follows the rule above. + still reuses its completed files. React V2 offers **Retry from failed step** for an + attempt with files only when a failed file can't be retried on its own and no file is + still being prepared (since **0.261.303**). Otherwise an attempt with files is + recovered one file at a time with **Retry file**, which never repeats plan steps or + withdraws an available file. A whole-run retry prepared through the retry API follows + the rule above. - **Steps that completed without an optional input.** When the retry runs that input's failed producer again, the steps that completed without it, and every step computed from them, run again so they can use what the first attempt missed. @@ -565,6 +585,17 @@ exception remains server-side in `__cause__`. | `ownership_lost` | The current execution no longer owns its guarded writes. | | `result_unavailable` | Ownership, producer or integrity validation failed, or a document the step reads as an input is denied or held; no preview may replace it. | +The settings part of a saved run's binding is the configuration its work ran +under. Since **0.261.303** it leaves out what the settings document holds that +isn't configuration, listed in `functions_settings_runtime_state.py`: storage +metadata that changes on every save (`_etag`, `_ts`, `_rid`, `_self`, +`_attachments`, `_settings_revision`, `id`) and runtime state that background +tasks save, such as the Cosmos DB throughput monitor's readings and scale history. +A save by one of those tasks no longer makes a waiting run or a retry report +`recovery_changed`; a configuration change by an administrator still does. A new +attempt also no longer restores the pending result an earlier attempt left for a +step it runs again. + Since **0.261.209**, external-source identity comes from the signed-in session, which a scheduler continuation doesn't have. A step, saved wait or final-answer check refused only for that reason (`external_identity_session_unavailable`) @@ -572,6 +603,16 @@ reports the step failure `external_session_required`, which asks the user to send the request again, instead of `result_unavailable`. The refusal code is logged as `sc_authority_reason`; it is never shown to the user. +Since **0.261.303**, publishing a file is authorized by the rendering service +that ran its attempt (`render_attempt_scope`), so a run started from the chat +publishes with the sign-in it started with. A file whose sources still can't be +checked without a session, such as one rendered in the background, fails as +`output_sign_in_required` with that reason, and its step reports +`file_sign_in_required`. Neither the automatic nor the file-only retry can help, +because both run in the background, so the chat offers **Retry from failed step**. +See the +[files from external results fix](../fixes/ORCHESTRATION_FILE_SIGN_IN_PUBLICATION_FIX.md). + Render authorization must translate `checkpoint_storage_unavailable` to its storage/service-unavailable path rather than returning `False` or raising an access-denied error. Unexpected exceptions from current-identity callbacks must diff --git a/docs/explanation/features/ORCHESTRATION_DELIVERABLES.md b/docs/explanation/features/ORCHESTRATION_DELIVERABLES.md index c93600acf..e3df585c9 100644 --- a/docs/explanation/features/ORCHESTRATION_DELIVERABLES.md +++ b/docs/explanation/features/ORCHESTRATION_DELIVERABLES.md @@ -375,9 +375,9 @@ limits. Running image steps in parallel is a possible follow-up. portraits, is not supported; reports link those sources instead. - Markdown and plain-text files do not contain images. - A plan generates at most four images. -- React V2 offers **Retry from failed step** only for an attempt without files, because - preparing a whole-run retry withdraws the attempt's available files. So a run that - delivered its file but missed a requested image offers no whole-run retry in the chat, - and **Retry file** cannot generate an image. Ask again to create a new plan. The - server-side retry itself delivers the image and the file; only the chat does not offer - it for such an attempt. +- React V2 offers **Retry from failed step** for an attempt with files only when a failed + file can't be retried on its own (since **0.261.303**), because preparing a whole-run + retry withdraws the attempt's available files. So a run that delivered its file but + missed a requested image offers no whole-run retry in the chat, and **Retry file** + cannot generate an image. Ask again to create a new plan. The server-side retry itself + delivers the image and the file; only the chat does not offer it for such an attempt. diff --git a/docs/explanation/fixes/ORCHESTRATION_FILE_SIGN_IN_PUBLICATION_FIX.md b/docs/explanation/fixes/ORCHESTRATION_FILE_SIGN_IN_PUBLICATION_FIX.md new file mode 100644 index 000000000..203825364 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_FILE_SIGN_IN_PUBLICATION_FIX.md @@ -0,0 +1,143 @@ +# Orchestration Files From External Results Failing Without a Reason Fix + +**Version: 0.261.303** + +Fixed in version: **0.261.303**, recorded in +`application/single_app/config.py`. + +This affects the React V2 branch (`paullizer-react-v2-ui`) and deployments built +from it since **0.261.209**. + +## Issue + +A request to create a Word report from a "Use an action" step's results gathered +its data and wrote the report, then replied: + +> Files: issued_refunds_shell_company_linked_entities_bank_analysis.docx: could not be created. +> +> The request was only partially completed. This operation could not complete. + +The file card showed **Failed** and "This file could not be created." Nothing +said why, the card offered no **Retry file**, and the recovery card offered no +**Retry from failed step**, only "Review each file separately". + +Production telemetry showed that the file rendered in 11 seconds and was refused +when it was published: + +- `[ORCHESTRATION_EXTERNAL_SOURCES] A step's source was refused.` for the + `action_invoke` step, with `sc_stage=external_source_read` and + `sc_authority_reason=external_identity_session_unavailable`. +- `[ORCHESTRATION_EXECUTOR] A file render attempt finished.` with + `sc_status=failed`, `sc_output_code=output_access_denied` and + `sc_error_cause_type=ResultUnavailableError`. + +## Root cause + +### Publication checked the file's sources without the user's sign-in + +Since **0.261.209**, retained external results (web search, linked pages, deep +research, agents and actions) are rechecked with the app roles from the user's +signed-in session. `build_external_identity_reader` captures those roles while +the request context exists, because the run continues on a worker thread. + +A render step's own service has those roles: it is built on the request thread +when the run starts. Publishing the file, however, goes through +`upload_generated_file_artifact_stream_for_user`, which authorizes the file's +lineage again through `load_orchestration_artifact_binding`. That function asked +the registered artifact factory for a **new** `build_orchestration_services(...)` +on the worker thread. With no request context there, its identity reader had no +roles, and the recheck of the action result failed closed with +`external_identity_session_unavailable`. Every file whose content came from an +external result was refused this way. + +### The failure lost its reason + +`_source_visibility_code` mapped the refusal to `output_access_denied`, which is +not retryable, so the file offered no **Retry file**. The saved file projection +says "This file could not be created." for every failure except a time limit, and +the render step reported the generic `step_failed`. + +### The chat hid the only retry that could work + +React V2 offers **Retry from failed step** only for attempts without files. A +file retry runs in the background scheduler, which never has a signed-in session, +so even a retryable file would have failed the same way. + +## Fix + +### The rendering service authorizes its own publication + +`OrchestrationRenderingService.render_attempt` runs inside the new +`render_attempt_scope(self)` from `functions_orchestration_artifacts.py`. While +the attempt runs, `_bound_service` uses that service for a publication check of +the same user and conversation instead of the factory. A run started from the +chat therefore publishes its files under the sign-in it started with. + +The scope is a `ContextVar`, so it never crosses into another thread or request. +A service for another conversation or user is never used, and outside a render +attempt nothing changes. + +### A file that needs a sign-in says so + +- A refusal caused only by the missing session is classified as the new output + code `output_sign_in_required`, wherever it surfaces: + `_source_visibility_code`, `output_failure`, and + `load_orchestration_artifact_binding`. It is not retried automatically or file + by file, because both run in the background. +- A failed file's saved projection now explains why it could not be created for + the common causes: a missing sign-in, unconfirmed access to its conversation or + results, a missing or changed result, a source under review, or file creation + being off. +- The render step reports the new failure `file_sign_in_required`, which asks the + user to select **Retry from failed step**. + +### Retry from failed step is offered for such a file + +React V2 now offers **Retry from failed step** for an attempt with files when a +failed file can't be retried on its own and no file is still being prepared. The +recovery card explains that the retry creates the plan's files again without +repeating completed plan steps. The retry runs from the signed-in chat, so it +can check and publish the file. + +## Files modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_orchestration_artifacts.py` | `render_attempt_scope`, `signed_in_session_required`, and `_bound_service` using the rendering attempt's service. | +| `application/single_app/functions_orchestration_rendering.py` | Scoped render attempts, `output_sign_in_required` classification and `render_failure_code`. | +| `application/single_app/functions_orchestration_output_store.py` | Failure and unavailable messages for the new code and common failures. | +| `application/single_app/functions_orchestration_continuation.py` | Scheduler reconciliation reports the same step failure. | +| `application/single_app/functions_orchestration_schema.py` | `file_sign_in_required` failure message. | +| `application/v2_ui/src/components/chat/OrchestrationRecoveryNotice.tsx` | **Retry from failed step** for a file that can't be retried alone. | +| `application/single_app/config.py` | Version 0.261.303. | + +## Testing + +- `functional_tests/test_orchestration_file_sign_in_publication.py` runs the + real upload and publication modules. A file publishes when the factory's service + has no sign-in, a foreign conversation's service is never used, and a refused + file fails as `output_sign_in_required` with its reason and no file-only retry. + Its step reports `file_sign_in_required`. +- `functional_tests/test_orchestration_output_lifecycle.py` now expects the saved + reason for an access refusal. +- `ui_tests/test_v2_orchestration_recovery.py` checks that a file that can't be + retried alone offers **Retry from failed step**, and that a retryable file still + offers only **Retry file**. + +Without the scope, the publication test fails exactly as production did: the file +ends `failed` and unavailable. + +## Impact + +| Before | After | +| --- | --- | +| A file built from web, deep research, agent or action results failed at publication in every orchestrated run. | It publishes with the sign-in the run started with. | +| The file card said only "This file could not be created." | It says why, for example that its results can only be checked with the user's sign-in. | +| No retry was offered. | **Retry from failed step** creates the files again from the chat, reusing completed steps. | + +## Known limitations + +A file rendered in the background, such as after a run waited for a long +computation, still can't check web, deep research, agent or action results. It +now fails with that reason, and **Retry from failed step** creates it from the +chat. diff --git a/docs/explanation/fixes/ORCHESTRATION_SETTINGS_WRITE_RECOVERY_FIX.md b/docs/explanation/fixes/ORCHESTRATION_SETTINGS_WRITE_RECOVERY_FIX.md new file mode 100644 index 000000000..06f5b2f87 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_SETTINGS_WRITE_RECOVERY_FIX.md @@ -0,0 +1,177 @@ +# Orchestration Runs Failing While Their Work Was Still Running Fix + +**Version: 0.261.303** + +Fixed in version: **0.261.303**, recorded in +`application/single_app/config.py`. + +This affects the React V2 branch (`paullizer-react-v2-ui`) and deployments built +from it. + +## Issue + +An orchestrated request that waited on a long tabular analysis replied: + +> The request could not be completed. +> +> Saved step inputs changed. Previously completed work will not be repeated. + +The plan panel disagreed with that reply. Its header said **The plan could not +complete**, but the analysis step still read **Waiting for results**, the answer +it fed still read **In progress** with a spinner, and the recovery card added +"Required computation is still pending. Its result is not ready to consume." No +**Retry from failed step** button was offered, so the run could never be +recovered. The analysis itself kept running and finished seconds later, with +nothing left to use its result. + +Production telemetry for the run, in UTC: + +| Time | Event | +| --- | --- | +| 14:24:06 | The native tabular analysis was queued: 1,800 rows in 4 batches. | +| 14:24:07 | The run's request ended with the run **waiting**, as designed. | +| 14:24:27 | All four analysis batches finished; the reduce phase started. | +| 14:24:30.598 | `[COSMOS_THROUGHPUT] Throughput update submitted.` scaled the `settings` container from 1,000 to 2,000 RU. | +| 14:24:30.766 | `[ASC] App settings updated and published successfully.` recorded that scale action in the settings document. | +| 14:24:34.147 | `[ORCHESTRATION_RUNS] Headless execution did not complete.` with `sc_error_type=CheckpointError` and `sc_execution_code=recovery_changed`. | +| about 14:24:36 | The analysis job finished. | + +In the previous 14 days, all three `recovery_changed` execution failures in this +deployment came after a settings save, two of them within 6 seconds of it. + +## Root cause + +### Every settings save looked like a policy change + +A saved run binds its checkpoints to the settings its work ran under, so a real +policy change, such as switching a capability off, can't reuse work produced +under the old policy. `functions_orchestration_checkpoints._execution_settings_fingerprint` +hashed the **whole** settings document into the run's `execution_binding` and into +every step's input fingerprint. + +That document isn't only configuration: + +- The settings store rewrites `_etag`, `_ts` and `_settings_revision` on every + save. +- Background tasks save their own runtime state into it: the Cosmos DB + throughput monitor (`cosmos_throughput_*` readings, scale history and + per-container scale timestamps), semantic search quota warnings + (`service_health`, saved by searches themselves), Control Center refresh + times, retention policy run times, the application release check, and the + log timers that turn debug logging off. + +So any save, by an administrator or a background task, changed the fingerprint. +The scheduler continues a waiting run by recomputing its binding with the current +settings. After the 14:24:30 save the binding no longer matched, and +`ContinuationCheckpoints` raised `CheckpointError('recovery_changed')`. The same +mismatch also refused **Retry from failed step** for any failed run once settings +had been saved since it ran. + +### An attempt that ended while a step waited was a dead end + +The continuation failed as a whole, before it reached any step. Run-level +failures publish the reply without touching the saved steps, so the waiting +analysis step and the answer step behind it stayed `waiting`. Then: + +- The V2 plan panel took its step badges and deliverable rows from those step + states alone, so a failed run still showed **Waiting for results** and + **In progress**. +- `recovery_projection` reported `result_not_ready` for any `waiting` step, and + `validate_resume` raised `result_not_ready` when preparing a retry. A failed + run with a leftover wait could therefore never be retried, although nothing + would ever finish that wait. + +### A retry would have restored the ended attempt's wait + +Each checkpoint saves every result its attempt held, including the pending +result of a step that was still waiting. A retry that reused a step completed in +the failed attempt restored that pending result too, so the new attempt saw two +producers for the step it was running again and stopped with `recovery_changed`. + +## Fix + +### Settings fingerprints ignore storage metadata and runtime state + +The new `functions_settings_runtime_state.py` lists what the settings document +holds that is not configuration: + +- storage fields: Cosmos DB metadata (`_etag`, `_ts`, `_rid`, `_self`, + `_attachments`, from `app_settings_store.COSMOS_METADATA_FIELDS`), + `_settings_revision` and `id`; +- runtime state that background tasks save: `service_health`, + `control_center_last_refresh`, `retention_policy_last_run`, + `retention_policy_next_run`, the release-check fields, every + `functions_logging_timers.LOGGING_TIMERS` key, and the `cosmos_throughput_` and + `control_center_auto_refresh_` key families. + +`_execution_settings_fingerprint` hashes the settings without those keys. +Configuration an administrator saves stays in the fingerprint, so a real policy +change still stops a waiting run and still refuses reuse. + +### An ended attempt no longer blocks recovery + +- `recovery_projection` reports `result_not_ready` only while the run itself is + still `waiting` or `running`. In a `failed` or `cancelled` run, a waiting step + is listed under **Execute on retry**. +- `validate_resume` runs such a step again in the new attempt instead of raising + `result_not_ready`. A same-attempt continuation is unchanged. +- `restore_context` doesn't restore another run's pending, output-less result or + its wait. Nothing could have read a pending result, and only its own attempt + could finish it. + +### The plan panel shows what an ended run left unfinished + +In the React V2 Run view, when the run has failed, been stopped or been +superseded: + +- a step still pending, running or waiting reads **Not finished**, with "The run + ended before this step finished." in place of its stale progress summary; +- a deliverable whose steps didn't finish reads **Not delivered** with "The run + ended before this was finished." instead of **In progress**. + +## Files modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_settings_runtime_state.py` | New registry of settings keys that are storage metadata or runtime state. | +| `application/single_app/functions_orchestration_checkpoints.py` | Fingerprints configuration only; doesn't restore another run's pending result. | +| `application/single_app/functions_orchestration_recovery.py` | Ended attempts with a waiting step are retryable. | +| `application/v2_ui/src/lib/orchestrationPlan.ts` | `stepUnfinishedByEndedRun`, and `deliverableRows` takes the run status. | +| `application/v2_ui/src/components/chat/OrchestrationRunView.tsx` | **Not finished** step badges and summaries. | +| `application/v2_ui/src/components/chat/OrchestrationDeliverables.tsx` | Passes the run status to the deliverable rows. | +| `application/single_app/config.py` | Version 0.261.303. | + +## Testing + +- `functional_tests/test_orchestration_settings_runtime_state_binding.py` + replays the incident: a waiting run continues after a simulated autoscaler save + and keeps its binding, while a policy change still stops it with + `recovery_changed`. It also parses each background settings writer and fails if + one saves a key the registry doesn't cover. +- `functional_tests/test_orchestration_terminal_wait_retry.py` covers a run that + failed or was stopped while waiting: retry is offered, the waiting step runs + again, completed work is reused, and an ended attempt's wait isn't restored in + either step order. A wait in a run that can still continue keeps blocking a + retry. +- `ui_tests/test_v2_orchestration_recovery.py` checks **Not finished**, + **Not delivered** and the retry button in Chromium. + +Without the fix, both binding regression tests and five of the terminal-wait tests +fail; the UI test fails without the plan panel changes. + +## Impact + +| Before | After | +| --- | --- | +| Any settings save, including a Cosmos DB scale action, failed every run waiting at that moment with "Saved step inputs changed". | Background saves don't affect saved runs. Only a configuration change by an administrator does. | +| The same saves refused **Retry from failed step** for earlier failed runs. | Retries reuse saved work unless configuration changed. | +| A failed run with a leftover wait showed **Waiting for results** and **In progress**, and offered no retry. | It shows **Not finished** and **Not delivered**, and offers **Retry from failed step**. | + +## Known limitations + +- Runs saved before this version carry bindings computed over the whole settings + document, so they no longer match after the upgrade. Their retry is refused with + "Saved step inputs changed", as it already was after any settings save. Ask again + instead. +- When a run fails while its analysis job is still running, the job still runs to + completion. Its result isn't used, and the retry starts a new job. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 57e628a2f..bb8a4e3a4 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,22 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.303)** + +#### Bug Fixes + +* **Orchestrated Runs Reported Failure While Their Work Was Still Running** + * A plan waiting on a long tabular analysis could reply "The request could not be completed. Saved step inputs changed." while the analysis was still running. The plan panel kept showing the step as **Waiting for results** and the answer as **In progress**, and offered no retry. A saved run's checkpoints were bound to a fingerprint of the whole settings document, so any settings save, including the Cosmos DB throughput autoscale recording a scale action, made the waiting run, and later retries of failed runs, look like their inputs had changed. + * The fingerprint now leaves out storage metadata and the runtime state background tasks save. A configuration change by an administrator still stops saved work from being reused. + * A run that fails or is stopped while a step is waiting shows that step as **Not finished** and its deliverable as **Not delivered**, and offers **Retry from failed step**, which runs the waiting step again and reuses completed ones. + * Runs saved before this version can't be retried after the upgrade; ask again. + * (Ref: `functions_settings_runtime_state.py`, `functions_orchestration_checkpoints.py`, `functions_orchestration_recovery.py`, `OrchestrationRunView.tsx`, `orchestrationPlan.ts`, [Orchestration Runs Failing While Their Work Was Still Running Fix](fixes/ORCHESTRATION_SETTINGS_WRITE_RECOVERY_FIX.md)) + +* **Files Built From Web, Agent or Action Results Failed Without a Reason** + * An orchestrated file whose content came from web search, linked pages, deep research, an agent or an action rendered, then failed with only "This file could not be created." and no retry. Publishing it rechecked its sources through a new service that didn't have the user's sign-in. Publication now uses the rendering service that ran the attempt, which has it. + * A file that still can't be checked without a sign-in, such as one prepared in the background, says so. A failed file now explains why it could not be created. When a failed file can't be retried on its own, the chat offers **Retry from failed step**. + * (Ref: `functions_orchestration_artifacts.py` `render_attempt_scope`, `functions_orchestration_rendering.py`, `functions_orchestration_output_store.py`, `OrchestrationRecoveryNotice.tsx`, [Orchestration Files From External Results Failing Without a Reason Fix](fixes/ORCHESTRATION_FILE_SIGN_IN_PUBLICATION_FIX.md)) + ### **(v0.261.302)** #### Bug Fixes diff --git a/docs/guides/create-files-with-orchestration.md b/docs/guides/create-files-with-orchestration.md index 40462a184..930cbb4b1 100644 --- a/docs/guides/create-files-with-orchestration.md +++ b/docs/guides/create-files-with-orchestration.md @@ -170,6 +170,15 @@ offers **Retry file** for each failed file rather than **Retry from failed step* so recovering one file never withdraws another that is already available. If other work in that attempt failed, ask again to create a new plan. +Since **0.261.303**, a failed file says why it could not be created. A file built +from web search, linked pages, deep research, an agent or an action is checked +with your sign-in before it's published. If it was prepared where your sign-in +wasn't available, for example after a run waited for a long computation, it says +so. File retries run in the background without your sign-in, so the conversation +offers **Retry from failed step** for such a file instead of **Retry file**. That +retry runs from the chat, reuses the completed steps and creates the plan's files +again. + Retrying cannot repair an unsupported format, invalid data or revoked source access. Restore the required access or revise the plan when its requirements have changed. An authorization or storage outage is not proof that a previously diff --git a/docs/guides/review-and-edit-orchestration-plans.md b/docs/guides/review-and-edit-orchestration-plans.md index 84e322822..087cd464c 100644 --- a/docs/guides/review-and-edit-orchestration-plans.md +++ b/docs/guides/review-and-edit-orchestration-plans.md @@ -366,6 +366,11 @@ prepared retry** are not offered for a waiting attempt. **Stop execution** continues to request cancellation from the server; closing the browser does not mean the computation stopped. +Since **0.261.303**, if the attempt fails or is stopped as a whole while a task +still waits, the plan shows that task as **Not finished** and what it produces as +**Not delivered**. Nothing will finish that wait any more, so **Retry from failed +step** is offered; it runs the waiting task again and reuses the completed ones. + The interface recognizes waiting without assuming that a downloadable file exists. Per-file publication and retry controls require their own server-owned output lifecycle; waiting alone does not enable them. @@ -408,7 +413,10 @@ remain valid downloads; a zero row, character, or byte count is not a failure. can be retried. It requests that file again from retained results, not another plan, producer task, or sibling output. An exhausted automatic-attempt count does not by itself authorize a manual retry. Non-retryable failures keep their -status and the server's explanation instead of offering a whole-plan replay. +status and the server's explanation. Since **0.261.303**, when a failed file +can't be retried on its own and no other file is still being prepared, the +recovery card offers **Retry from failed step**, which creates the plan's files +again without repeating completed plan steps. If a retry response is lost, **Retry same request** keeps the original action identity. The identity is saved in this browser tab before the request and diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md index cca942a46..b7ac311e8 100644 --- a/docs/reference/chat-controls.md +++ b/docs/reference/chat-controls.md @@ -538,7 +538,7 @@ saved effective plan, not the current composer selections or a new planner call. | Control | What it does | Why you would use it | Enabled by | | --- | --- | --- | --- | -| Retry from failed step | Creates a linked attempt that restores valid completed-step results and executes the incomplete work. | Recover after a failure without repeating successful plan steps or duplicating the question. | `enable_chat_orchestration`, current access, and recoverable saved checkpoints; not a substitute for individual file recovery | +| Retry from failed step | Creates a linked attempt that restores valid completed-step results and executes the incomplete work. | Recover after a failure without repeating successful plan steps or duplicating the question. | `enable_chat_orchestration`, current access, and recoverable saved checkpoints. For an attempt with files, only when a failed file can't be retried on its own and no file is still being prepared (since **0.261.303**) | | Confirm retry / Cancel | Confirms the possible external effects of retrying a failed agent/action, or dismisses the confirmation without executing it. Since **0.261.212** it also appears for a step that starts a saved workflow; its retry never starts the workflow twice, and links the run when the plan already started it. | Decide whether it is safe to repeat the failed step's internal tool activity. | A recoverable attempt that requires external-effect confirmation | | Run prepared retry | Starts a recovery attempt that was already prepared but has not executed. | Continue after preparation was saved but execution was interrupted by navigation or connection loss. | An unstarted saved recovery attempt | | Review saved attempt | Opens the selected attempt in the Plan/Run view. | Inspect the failure, completed steps, and remaining work without editing or rerunning history. | A saved orchestration attempt | @@ -555,6 +555,12 @@ attempt active. Reload and status checks do not execute it again, and waiting does not expose a run-retry button or a completed-file link. File-specific publication and retry controls require the server's separate output lifecycle. +Since **0.261.303**, an attempt that fails or is stopped as a whole while a step +is still pending, running or waiting shows that step as **Not finished** and the +deliverable it produces as **Not delivered**, instead of **Waiting for results** +or **In progress**. Nothing will finish that work in the ended attempt, so +**Retry from failed step** runs it again. + Stop requests cancellation on the server. A connection loss instead requires checking the existing execution; it must not automatically start another one. When a checkpoint or source is unavailable, the interface explains why recovery @@ -583,6 +589,11 @@ survive reload in the same browser tab, but reloading never posts a retry or reruns the original plan. An expired sign-in or a conflict requires checking saved state before another request. +Since **0.261.303**, a failed file says why it could not be created, for example +that the results it uses can only be checked with your sign-in. File retries run +in the background, where that sign-in isn't available, so such a file offers +**Retry from failed step** instead of **Retry file**. + Generated-file history entries are informational, not uploaded-file previews or download receipts. Unavailable history entries show a safe explanation and close any cached preview. Empty TXT and MD files can still be downloaded; zero counts diff --git a/functional_tests/test_orchestration_file_sign_in_publication.py b/functional_tests/test_orchestration_file_sign_in_publication.py new file mode 100644 index 000000000..b3a12781d --- /dev/null +++ b/functional_tests/test_orchestration_file_sign_in_publication.py @@ -0,0 +1,129 @@ +# test_orchestration_file_sign_in_publication.py +"""A file a signed-in run renders is published under that run's sign-in, or says why not. + +Version: 0.261.303 +Implemented in: 0.261.303 + +A Word report rendered from an action's results failed with only "This file could not be +created." Publishing the file re-checked its sources through a fresh rendering service built on +the execution's worker thread. That service had no signed-in session, so every web search, web +page, deep research, agent or action source in the file's lineage was refused. Publication is +now checked by the rendering service that ran the attempt. A file that still cannot be checked +without a sign-in, such as one rendered in the background, fails as ``output_sign_in_required`` +with a reason and no file-only retry, and its step reports ``file_sign_in_required``. Real +production modules run with external Azure I/O doubled. +""" + +import pytest + +from functions_orchestration_artifacts import OUTPUT_SIGN_IN_REQUIRED, signed_in_session_required +from functions_orchestration_output_store import ( + OUTPUT_FAILURE_MESSAGES, + OUTPUT_UNAVAILABLE_MESSAGES, + OutputError, + OutputUnavailableError, +) +from functions_orchestration_rendering import _render_step_result, output_failure +from functions_orchestration_result_contracts import EXTERNAL_SESSION_UNAVAILABLE_REASON +from functions_orchestration_results import ResultUnavailableError +from test_orchestration_output_lifecycle import lifecycle, production_modules # noqa: F401 +from test_support.versioning import assert_app_version_at_least + + +def _no_session(record, *, operation): + raise ResultUnavailableError(EXTERNAL_SESSION_UNAVAILABLE_REASON) + + +def test_version_includes_the_file_sign_in_fix(): + assert_app_version_at_least('0.261.303') + + +def test_publication_is_checked_by_the_service_that_rendered_the_file(lifecycle, monkeypatch): + import functions_orchestration_artifacts as artifacts + + attempt_service = lifecycle.service + # What the registered factory built on a worker thread: no signed-in session to check sources. + sessionless = lifecycle.restart() + lifecycle.service = attempt_service + sessionless.authorize_execution = _no_session + monkeypatch.setattr(artifacts, '_service_factory', lambda user_id, conversation_id: sessionless) + + completed = lifecycle.run(lifecycle.prepare()) + assert completed['state'] == 'completed', completed + assert lifecycle.blobs.uploads == 1 + + # Once the attempt ends, a publication check uses a fresh service again, and a refusal + # caused only by the missing sign-in keeps that reason. + message = attempt_service.transport.message(lifecycle.raw(completed), committed=True) + with pytest.raises(OutputUnavailableError) as refused: + lifecycle.modules.sources.authorize_generated_artifact_source('owner', message, for_publication=True) + assert refused.value.code == OUTPUT_SIGN_IN_REQUIRED + + +def test_another_conversations_attempt_never_authorizes_this_publication(lifecycle, monkeypatch): + import functions_orchestration_artifacts as artifacts + + completed = lifecycle.run(lifecycle.prepare()) + message = lifecycle.service.transport.message(lifecycle.raw(completed), committed=True) + sessionless = lifecycle.restart() + sessionless.authorize_execution = _no_session + monkeypatch.setattr(artifacts, '_service_factory', lambda user_id, conversation_id: sessionless) + foreign = type('Foreign', (), {})() + foreign.store = type('Store', (), {'user_id': 'owner', 'conversation_id': 'conversation-2'})() + with artifacts.render_attempt_scope(foreign): + with pytest.raises(OutputUnavailableError) as refused: + lifecycle.modules.sources.authorize_generated_artifact_source('owner', message, for_publication=True) + assert refused.value.code == OUTPUT_SIGN_IN_REQUIRED + + +def test_a_file_that_needs_a_sign_in_says_so_and_offers_no_file_only_retry(lifecycle, production_modules): + service = lifecycle.service + real = service.authorize_execution + + def background(record, *, operation): + # A render running in the background: checking its sources needs a sign-in it lacks. + if operation == 'prepare': + raise ResultUnavailableError(EXTERNAL_SESSION_UNAVAILABLE_REASON) + return real(record, operation=operation) + + service.authorize_execution = background + output = lifecycle.prepare() + failed = lifecycle.run(output) + assert failed['state'] == 'failed' + assert failed['error_code'] == OUTPUT_SIGN_IN_REQUIRED + assert failed['message'] == OUTPUT_UNAVAILABLE_MESSAGES[OUTPUT_SIGN_IN_REQUIRED] + assert failed['can_retry'] is False and failed['next_retry_at'] is None + assert lifecycle.raw(output)['automatic_attempts'] == 1 + assert lifecycle.blobs.uploads == 0 + + # Read from a signed-in request, the saved file says why it could not be created. + service.authorize_execution = real + saved = service.read(output['output_id']) + assert saved['state'] == 'failed' and saved['available'] is True + assert saved['error_code'] == OUTPUT_SIGN_IN_REQUIRED + assert saved['message'] == OUTPUT_FAILURE_MESSAGES[OUTPUT_SIGN_IN_REQUIRED] + assert saved['can_retry'] is False + with pytest.raises(OutputError): + service.manual_retry(output['output_id'], 'file-only-retry') + + schema = production_modules.schema + step = _render_step_result( + saved, None, build_step_result=schema.build_step_result, build_failure=schema.build_failure, + ) + assert step['status'] == 'failed' + assert step['failure']['code'] == 'file_sign_in_required' + assert step['failure']['message'] == schema.FAILURE_MESSAGES['file_sign_in_required'] + assert step['summary'] == saved['message'] + + +def test_only_a_missing_session_is_classified_as_a_sign_in_requirement(): + refused = ResultUnavailableError(EXTERNAL_SESSION_UNAVAILABLE_REASON) + wrapped = OutputUnavailableError('output_access_denied') + wrapped.__cause__ = refused + assert signed_in_session_required(refused) and signed_in_session_required(wrapped) + assert output_failure(refused) == (OUTPUT_SIGN_IN_REQUIRED, False) + assert output_failure(wrapped) == (OUTPUT_SIGN_IN_REQUIRED, False) + + denied = ResultUnavailableError('external_identity_access_denied') + assert not signed_in_session_required(denied) + assert output_failure(denied) == ('output_access_denied', False) diff --git a/functional_tests/test_orchestration_output_lifecycle.py b/functional_tests/test_orchestration_output_lifecycle.py index 150ed9560..3cf094f00 100644 --- a/functional_tests/test_orchestration_output_lifecycle.py +++ b/functional_tests/test_orchestration_output_lifecycle.py @@ -1,10 +1,11 @@ # test_orchestration_output_lifecycle.py """ Real retained-result/render/transport/commit/download lifecycle integration. -Version: 0.261.245 +Version: 0.261.303 Implemented in: 0.261.127 Container-only generated-file access covered in: 0.261.232 Merged documents rendered from their own lineage added in: 0.261.245 +Failed files say why they could not be created in: 0.261.303 Production modules (including the complete upload and download modules) run with external Azure I/O doubled. No AST-extracted service, model call, or provider is @@ -48,6 +49,7 @@ ) from functions_orchestration_output_store import ( MAX_AUTOMATIC_ATTEMPTS, + OUTPUT_FAILURE_MESSAGES, OUTPUT_RECORD_TYPE, OrchestrationOutputStore, OutputConflictError, @@ -1522,8 +1524,9 @@ def test_nonretryable_failures_never_schedule_automatic_or_manual_replay(lifecyc with pytest.raises(OutputError): lifecycle.service.manual_retry(output["output_id"], "invalid-retry") duplicate = lifecycle.run(output) + # The saved projection says why an access refusal stopped the file (0.261.303). expected = { - **failed, "available": True, "message": "This file could not be created.", + **failed, "available": True, "message": OUTPUT_FAILURE_MESSAGES["output_access_denied"], } if isinstance(failure, PermissionError) else failed assert duplicate == expected and len(lifecycle.render_calls) == 1 assert "private" not in json.dumps(failed) diff --git a/functional_tests/test_orchestration_settings_runtime_state_binding.py b/functional_tests/test_orchestration_settings_runtime_state_binding.py new file mode 100644 index 000000000..bdd45a2ed --- /dev/null +++ b/functional_tests/test_orchestration_settings_runtime_state_binding.py @@ -0,0 +1,203 @@ +# test_orchestration_settings_runtime_state_binding.py +"""Background settings writes never invalidate saved orchestration progress. + +Version: 0.261.303 +Implemented in: 0.261.303 + +A waiting run failed with "Saved step inputs changed" because its execution binding hashed the +whole settings document. That document also holds storage metadata that changes on every save +and runtime state that background tasks write, such as the Cosmos DB throughput monitor's +readings. These tests prove those fields no longer change a saved binding, that real policy +changes still do, and that every background writer's keys are covered. Real fingerprints, +leases and waiting checkpoints run; only storage transport and producer/model I/O are isolated. +""" + +import ast +from pathlib import Path + +import pytest + +from test_orchestration_dependency_recovery import durable # noqa: F401 +from test_orchestration_dependency_runtime import compose, runtime # noqa: F401 +from test_orchestration_waiting_continuation import claim, start_waiting +from test_support.versioning import assert_app_version_at_least + + +APP = Path(__file__).resolve().parents[1] / 'application' / 'single_app' + +# What a settings save and a Cosmos DB throughput scale-up change, as seen at 14:24:30 UTC on +# 2026-10-08 just before a waiting run failed. None of it is configuration. +BEFORE_WRITE = { + '_etag': '"etag-before"', '_ts': 1791469400, '_rid': 'rid', '_self': 'self', '_attachments': 'attachments/', + '_settings_revision': 41, 'id': 'app_settings', + 'cosmos_throughput_last_checked_at': '2026-10-08T14:22:20+00:00', + 'cosmos_throughput_last_observed_percent': 71, + 'cosmos_throughput_cached_status': {'mode': 'autoscale', 'current_ru': 1000}, + 'cosmos_throughput_container_policies': {'settings': {'min_ru': 1000}}, + 'service_health': {'semantic_search': {'status': 'healthy'}}, + 'control_center_last_refresh': '2026-10-08T06:00:00+00:00', + 'enable_debug_logging': True, 'debug_logging_turnoff_time': '2026-10-08T15:00:00+00:00', +} +AFTER_WRITE = { + '_etag': '"etag-after"', '_ts': 1791469470, '_settings_revision': 42, + 'cosmos_throughput_last_checked_at': '2026-10-08T14:24:30+00:00', + 'cosmos_throughput_last_observed_percent': 96, + 'cosmos_throughput_last_scale_action': 'up', 'cosmos_throughput_last_scale_to_ru': 2000, + 'cosmos_throughput_cached_status': {'mode': 'autoscale', 'current_ru': 2000}, + 'cosmos_throughput_container_policies': { + 'settings': {'min_ru': 1000, 'last_scale_up_at': '2026-10-08T14:24:30+00:00'}, + }, + 'service_health': {'semantic_search': {'status': 'quota_exceeded'}}, + 'control_center_last_refresh': '2026-10-08T14:24:00+00:00', + 'control_center_auto_refresh_next_run': '2026-10-09T06:00:00+00:00', + 'retention_policy_last_run': '2026-10-08T14:00:00+00:00', + 'last_update_check_time': '2026-10-08T14:24:00+00:00', 'update_available': True, + 'enable_debug_logging': False, 'debug_logging_turnoff_time': None, +} +POLICY_CHANGES = [ + {'chat_orchestration_max_steps': 4}, + {'chat_orchestration_enabled_capabilities': ['document_search']}, + {'enable_user_workspace': False}, + {'enable_chat_orchestration_actions': True}, +] + +# Background tasks that write settings, and the dictionaries they save. The keys these build +# must all be runtime state, so a new runtime key cannot silently invalidate saved progress. +SETTINGS_WRITERS = ( + ('functions_cosmos_throughput.py', 'build_runtime_update', ('update',)), + ('background_tasks.py', '_seed_control_center_auto_refresh_next_run', ()), + ('functions_control_center.py', 'execute_control_center_refresh', ('settings_updates',)), + ('functions_retention_policy.py', 'execute_retention_policy', ('settings_updates',)), + ('functions_service_health.py', 'record_semantic_search_quota_exceeded', ()), + ('functions_service_health.py', 'clear_semantic_search_quota_warning', ()), + ('functions_settings.py', 'get_application_update_status', ('updates',)), +) + + +def _bindings(runtime, case, settings): + checkpoints = runtime.checkpoints + binding = checkpoints.context_binding(case.context, case.plan, settings) + step = checkpoints.step_input_fingerprint(case.plan['steps'][0], case.context, binding, settings=settings) + return binding, step + + +def _string_keys(node): + return [key.value for key in node.keys if isinstance(key, ast.Constant) and isinstance(key.value, str)] + + +def _saved_keys(file_name, function_name, names): + tree = ast.parse((APP / file_name).read_text(encoding='utf-8')) + function = next( + node for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == function_name + ) + keys = set() + for node in ast.walk(function): + if isinstance(node, ast.Call): + target = node.func + if isinstance(target, ast.Name) and target.id == 'update_settings': + for argument in node.args[:1]: + if isinstance(argument, ast.Dict): + keys.update(_string_keys(argument)) + if ( + isinstance(target, ast.Attribute) and target.attr == 'update' + and isinstance(target.value, ast.Name) and target.value.id in names + ): + for argument in node.args: + if isinstance(argument, ast.Dict): + keys.update(_string_keys(argument)) + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id in names and isinstance(node.value, ast.Dict): + keys.update(_string_keys(node.value)) + if ( + isinstance(target, ast.Subscript) and isinstance(target.value, ast.Name) + and target.value.id in names and isinstance(target.slice, ast.Constant) + and isinstance(target.slice.value, str) + ): + keys.add(target.slice.value) + return keys + + +def test_version_includes_the_runtime_state_binding_fix(): + assert_app_version_at_least('0.261.303') + + +def test_storage_metadata_and_runtime_state_leave_saved_bindings_unchanged(runtime): + case = runtime.make([compose()]) + before = {**case.settings, **BEFORE_WRITE} + after = {**before, **AFTER_WRITE} + assert _bindings(runtime, case, after) == _bindings(runtime, case, before) + assert _bindings(runtime, case, before) == _bindings(runtime, case, case.settings) + + +@pytest.mark.parametrize('changes', POLICY_CHANGES) +def test_policy_changes_still_invalidate_saved_bindings_beside_runtime_state(runtime, changes): + case = runtime.make([compose()]) + before = {**case.settings, **BEFORE_WRITE} + changed = {**before, **AFTER_WRITE, **changes} + original_binding, original_step = _bindings(runtime, case, before) + changed_binding, changed_step = _bindings(runtime, case, changed) + assert changed_binding != original_binding + assert changed_step != original_step + + +def test_registry_matches_the_settings_store_and_logging_timers(runtime): + import app_settings_store + import functions_logging_timers + import functions_settings_runtime_state as registry + + assert registry.SETTINGS_STORAGE_KEYS == frozenset({ + *app_settings_store.COSMOS_METADATA_FIELDS, app_settings_store.SETTINGS_REVISION_FIELD, 'id', + }) + for timer in functions_logging_timers.LOGGING_TIMERS.values(): + for key in timer.values(): + assert registry.is_settings_runtime_state_key(key), key + # Configuration an administrator saves stays in the fingerprint. + for key in ('enable_chat_orchestration', 'chat_orchestration_max_steps', 'enable_user_workspace', 'gpt_model'): + assert not registry.is_settings_runtime_state_key(key), key + assert not registry.is_settings_runtime_state_key(None) + + +@pytest.mark.parametrize('file_name,function_name,names', SETTINGS_WRITERS) +def test_every_background_settings_writer_saves_only_runtime_state(runtime, file_name, function_name, names): + import functions_settings_runtime_state as registry + + keys = _saved_keys(file_name, function_name, names) + assert keys, f'{function_name} no longer saves recognizable settings keys; update this test.' + assert sorted(key for key in keys if not registry.is_settings_runtime_state_key(key)) == [] + + +def test_a_waiting_run_continues_after_a_background_settings_write(durable): + settings = durable.case.settings + settings.update(BEFORE_WRITE) + start_waiting(durable) + original = durable.read_run('run-1') + + settings.update(AFTER_WRITE) + acquired = claim(durable, submission='after-background-write') + assert acquired['acquired'] is True + record = acquired['record'] + result = durable.run(record, durable.fresh_context(record)) + saved = durable.read_run('run-1') + + assert result['status'] == 'waiting' + assert saved['status'] == 'waiting' + assert saved['execution_binding'] == original['execution_binding'] + assert saved['task_results'] == original['task_results'] + assert saved['pending_results'] == original['pending_results'] + assert saved['execution_deadline_at'] == original['execution_deadline_at'] + + +def test_a_policy_change_still_stops_a_waiting_run(durable): + settings = durable.case.settings + settings.update(BEFORE_WRITE) + start_waiting(durable) + + settings.update({**AFTER_WRITE, 'chat_orchestration_max_steps': 4}) + acquired = claim(durable, submission='after-policy-change') + assert acquired['acquired'] is True + record = acquired['record'] + with pytest.raises(durable.runtime.checkpoints.CheckpointError) as failure: + durable.run(record, durable.fresh_context(record)) + assert failure.value.code == 'recovery_changed' diff --git a/functional_tests/test_orchestration_terminal_wait_retry.py b/functional_tests/test_orchestration_terminal_wait_retry.py new file mode 100644 index 000000000..7b1cab477 --- /dev/null +++ b/functional_tests/test_orchestration_terminal_wait_retry.py @@ -0,0 +1,142 @@ +# test_orchestration_terminal_wait_retry.py +"""A run that ends while a step still waits can be retried, and the wait runs again. + +Version: 0.261.303 +Implemented in: 0.261.303 + +A waiting run whose continuation failed as a whole kept its waiting step. The recovery +projection then reported "Required computation is still pending" and preparing a retry raised +``result_not_ready``, so the failed run could never be retried. Once the attempt has ended, +nothing will finish its wait: a retry now runs the waiting step again and reuses only completed +work. A wait in a run that can still continue keeps blocking a retry. Real leases, checkpoints +and retained results run; only storage transport and producer/model I/O are isolated. +""" + +from types import SimpleNamespace + +import pytest + +from test_orchestration_dependency_recovery import durable # noqa: F401 +from test_orchestration_dependency_runtime import runtime # noqa: F401 +from test_orchestration_waiting_continuation import start_waiting +from test_support.versioning import assert_app_version_at_least + + +def _snapshot_context(run_id): + return SimpleNamespace( + run_id=run_id, user_message='Explain the flagged filing', memory_context={}, selected_document_ids=[], + plan_contract_version=2, task_results={}, result_aliases={}, pending_results={}, + execution_deadline_at='2030-01-01T00:00:00+00:00', composition_profiles={}, + ) + + +def _pending_task(runtime, run_id): + producer = runtime.contracts.ProducerIdentity( + 'owner', 'conversation-1', run_id, 1, 'waited', 'tabular_analyze', 'tabular-analyze-v1', + ) + return runtime.contracts.TaskResult(producer, 'reason', 'pending', ()) + + +def _snapshot_with_wait(runtime, run_id): + checkpoints = runtime.checkpoints + state = checkpoints.context_state(_snapshot_context(run_id)) + state['task_results'] = {'waited': _pending_task(runtime, run_id).to_dict()} + state['pending_results'] = {'waited': {'kind': 'native_tabular_compute', 'job_id': 'old-attempt-job'}} + return {'state': state} + + +def _fail_while_waiting(durable, status='failed'): + """End the waiting attempt the way a failed continuation publishes it.""" + start_waiting(durable) + record = durable.read_run('run-1') + failure = durable.runtime.schema.build_failure('recovery_changed') + return durable.recovery._replace(record, { + 'status': status, 'outcome': status, 'failure': failure, 'failures': [failure], + 'error': failure['message'], 'execution_lease': None, + 'finalization_status': 'saved', 'message_saved': True, + }) + + +def test_version_includes_the_terminal_wait_retry_fix(): + assert_app_version_at_least('0.261.303') + + +@pytest.mark.parametrize('status', ['failed', 'cancelled']) +def test_an_ended_attempt_offers_retry_that_runs_its_wait_again(durable, status): + ended = _fail_while_waiting(durable, status) + reconciled = durable.recovery.reconcile_checkpoints(ended, lambda: True) + states = {step['step_id']: step['status'] for step in reconciled['execution_steps']} + assert states['failed'] == 'waiting' + + projection = durable.recovery.recovery_projection(reconciled) + assert projection['eligible'] is True + assert projection['reason_code'] is None + assert projection['reused_step_ids'] == ['retained'] + assert projection['retry_step_ids'] == ['failed', 'dependent'] + + payloads = durable.recovery.validate_resume( + ended, durable.fresh_context(ended), durable.case.settings, lambda: True, source_run_id=ended['id'], + ) + assert list(payloads) == ['retained'] + + +def test_the_retry_reuses_completed_work_and_finishes_the_interrupted_step(durable): + original = _fail_while_waiting(durable) + child = durable.prepare() + assert child['retry_of_run_id'] == 'run-1' + assert child['retry_reused_step_ids'] == ['retained'] + assert 'pending_results' not in child and 'task_results' not in child + + child = durable.recovery._replace(child, {'status': 'running', 'execution_lease': durable.recovery.lease_fields()}) + durable.case.model.replies.extend(['Interrupted work finished this time.', 'Answer from both results.']) + result = durable.run(child, durable.fresh_context(child)) + assert result['status'] == 'completed', result + assert [step['step_id'] for step in result['steps'] if step.get('reused')] == ['retained'] + assert result['pending_results'] == {} + assert result['task_results']['retained'] == original['task_results']['retained'] + parent = durable.read_run('run-1') + assert parent['latest_attempt_run_id'] == child['id'] + assert parent['status'] == 'failed' + + +def test_a_wait_in_a_run_that_can_continue_still_blocks_retry(durable): + start_waiting(durable) + record = durable.read_run('run-1') + assert record['status'] == 'waiting' + reconciled = durable.recovery.reconcile_checkpoints(record, lambda: True) + projection = durable.recovery.recovery_projection(reconciled) + assert projection['eligible'] is False + assert projection['reason_code'] == 'result_not_ready' + with pytest.raises(durable.runtime.checkpoints.CheckpointError) as failure: + durable.recovery.validate_resume( + record, durable.fresh_context(record), durable.case.settings, lambda: True, + ) + assert failure.value.code == 'result_not_ready' + + +def test_a_new_attempt_does_not_restore_an_ended_attempts_wait(runtime): + # Restored before the new attempt reaches the step: it must not wait on the old job. + context = _snapshot_context('run-2') + runtime.checkpoints.restore_context(context, _snapshot_with_wait(runtime, 'run-1')) + assert context.task_results == {} and context.pending_results == {} + + +def test_a_new_attempt_keeps_its_own_result_for_a_step_that_waited_before(runtime): + # Restored after the new attempt reached the step again: its own result stands. + context = _snapshot_context('run-2') + own = runtime.contracts.TaskResult( + runtime.contracts.ProducerIdentity( + 'owner', 'conversation-1', 'run-2', 2, 'waited', 'tabular_analyze', 'tabular-analyze-v1', + ), 'reason', 'pending', (), + ) + context.task_results['waited'] = own + runtime.checkpoints.restore_context(context, _snapshot_with_wait(runtime, 'run-1')) + assert context.task_results == {'waited': own} + assert context.pending_results == {} + + +def test_the_same_attempt_still_restores_its_own_wait(runtime): + context = _snapshot_context('run-1') + runtime.checkpoints.restore_context(context, _snapshot_with_wait(runtime, 'run-1')) + assert context.task_results['waited'] == _pending_task(runtime, 'run-1') + assert context.pending_results == {'waited': {'kind': 'native_tabular_compute', 'job_id': 'old-attempt-job'}} diff --git a/ui_tests/test_v2_orchestration_recovery.py b/ui_tests/test_v2_orchestration_recovery.py index 1dbedeb01..46862d662 100644 --- a/ui_tests/test_v2_orchestration_recovery.py +++ b/ui_tests/test_v2_orchestration_recovery.py @@ -1,11 +1,13 @@ # test_v2_orchestration_recovery.py """ Real-component browser coverage for orchestration failure and checkpoint recovery. -Version: 0.261.238 +Version: 0.261.303 Implemented in: 0.261.105 Earlier-version run refusals covered in: 0.261.139 Settled runs whose stored plan still reads running covered in: 0.261.141 Configurable step failure, reused by Microsoft 365 recovery coverage, in: 0.261.238 +Ended runs show unfinished steps and retry them; files that cannot be retried alone offer +Retry from failed step, in: 0.261.303 The production controller, SSE reader, stores, message list, and Run drawer execute in the existing local/Azure Playwright harness. Only API responses are deterministic. @@ -785,3 +787,96 @@ def test_legacy_refusal_at_run_start_puts_the_plan_away_with_the_server_message( assert state["streamError"] == LEGACY_PLAN_MESSAGE assert not has_active_orchestration(page) assert len(api.calls("/run")) == 1 + + +# A run can fail as a whole while a step still waits on a long computation: the continuation +# that would have finished it ended the attempt instead. Nothing will finish those steps now. +WAIT_FAILURE = { + "code": "recovery_changed", + "message": "Saved step inputs changed. Previously completed work will not be repeated.", +} +SIGN_IN_FILE_MESSAGE = ( + "This file could not be created because the results it uses can only be checked with your " + "sign-in, which wasn't available where the file was prepared. Retry from this chat to create it." +) + + +def interrupt_waiting_run(api): + """Save a failed run whose research step and answer were still waiting when it ended.""" + run_id = api.plan["run_id"] + api.failure = dict(WAIT_FAILURE) + api.finish(run_id, "failed") + api.steps[run_id] = [ + {"step_id": "read", "step_index": 0, "status": "completed", "summary": "Read saved reports", + "checkpoint_available": True}, + {"step_id": "research", "step_index": 1, "status": "waiting", + "summary": "Native computation is pending; no result is ready to consume."}, + {"step_id": "answer", "step_index": 2, "status": "waiting", "summary": "Waiting for required results."}, + ] + return run_id + + +def fail_file(api, *, can_retry): + """Save a partial run whose Word file failed, retryable on its own or not.""" + run_id = api.plan["run_id"] + api.finish(run_id, "failed") + record = api.records[run_id] + output = { + "output_id": "orender_" + "a" * 64, "step_id": "answer", "file_name": "bank_analysis.docx", + "output_format": "docx", "profile": "prepared_report_v1", "state": "failed", "available": True, + "attempt_count": 1, "automatic_attempts": 1, "max_automatic_attempts": 3, "next_retry_at": None, + "can_retry": can_retry, + "error_code": "output_storage_unavailable" if can_retry else "output_sign_in_required", + "message": "This file could not be created." if can_retry else SIGN_IN_FILE_MESSAGE, + "artifact_message_id": None, "row_count": None, "character_count": None, "size_bytes": None, + } + record.update(outcome="partial", outputs=[output]) + for message in api.messages: + if message["id"] == record["assistant_message_id"]: + message["metadata"]["orchestration"].update(outcome="partial", outputs=[copy.deepcopy(output)]) + return output + + +def test_ended_run_shows_unfinished_steps_as_not_finished_and_offers_retry(recovery_ui): + page, api = recovery_ui + interrupt_waiting_run(api) + mount_recovery(page, api, saved=True) + expect(page.get_by_text(WAIT_FAILURE["message"], exact=True).first).to_be_visible() + expect(page.get_by_role("button", name="Retry from failed step").first).to_be_enabled() + page.get_by_role("button", name="Review saved attempt").first.click() + drawer = page.get_by_role("complementary", name="Review drawer") + research = drawer.locator('[data-step-id="research"]') + expect(research.locator('[data-step-status="unfinished"]')).to_have_text("Not finished") + expect(research.get_by_text("The run ended before this step finished.", exact=True)).to_be_visible() + expect(drawer.locator('[data-step-id="answer"] [data-step-status="unfinished"]')).to_have_text("Not finished") + expect(drawer.locator('[data-step-id="read"] [data-step-status="completed"]')).to_be_visible() + expect(drawer.get_by_text("Waiting for results", exact=True)).to_have_count(0) + expect(drawer.get_by_text("Native computation is pending; no result is ready to consume.", exact=True)).to_have_count(0) + answer = drawer.locator('[data-deliverable-id="answer"]') + expect(answer).to_have_attribute("data-deliverable-state", "not_delivered") + expect(answer).to_contain_text("The run ended before this was finished.") + expect(drawer.get_by_role("button", name="Retry from failed step")).to_be_enabled() + expect(drawer.locator("p").filter(has_text="Execute on retry:")).to_contain_text("Ask the selected agent") + assert not api.calls("/run") and not api.calls("/retry") + + +def test_file_that_cannot_be_retried_alone_offers_retry_from_failed_step(recovery_ui): + page, api = recovery_ui + fail_file(api, can_retry=False) + mount_recovery(page, api, saved=True) + expect(page.get_by_text(SIGN_IN_FILE_MESSAGE, exact=True).first).to_be_visible() + expect(page.get_by_text("A file could not be created and cannot be retried on its own.", exact=False).first).to_be_visible() + expect(page.get_by_role("button", name="Retry from failed step").first).to_be_enabled() + expect(page.get_by_text("Review each file separately.", exact=False)).to_have_count(0) + expect(page.get_by_role("button", name="Retry file bank_analysis.docx")).to_have_count(0) + assert not api.calls("/run") and not api.calls("/retry") + + +def test_file_that_can_be_retried_alone_keeps_only_the_file_retry(recovery_ui): + page, api = recovery_ui + fail_file(api, can_retry=True) + mount_recovery(page, api, saved=True) + expect(page.get_by_role("button", name="Retry file bank_analysis.docx").first).to_be_visible() + expect(page.get_by_text("Review each file separately.", exact=False).first).to_be_visible() + expect(page.get_by_role("button", name="Retry from failed step")).to_have_count(0) + assert not api.calls("/run") and not api.calls("/retry")