diff --git a/application/single_app/config.py b/application/single_app/config.py index accbb4e12..777393c97 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.299" +VERSION = "0.261.300" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_m365_agent_continuation.py b/application/single_app/functions_m365_agent_continuation.py index 8a7fd87d1..5be41c4fd 100644 --- a/application/single_app/functions_m365_agent_continuation.py +++ b/application/single_app/functions_m365_agent_continuation.py @@ -1,7 +1,7 @@ # functions_m365_agent_continuation.py """Checkpoint real agent tool history so an approval does not repeat completed calls.""" -from contextlib import aclosing +from contextlib import aclosing, contextmanager from contextvars import ContextVar from dataclasses import asdict from functools import wraps @@ -14,10 +14,11 @@ from semantic_kernel.agents import ChatHistoryAgentThread from semantic_kernel.contents import AuthorRole, ChatHistory, ChatMessageContent, FunctionCallContent, FunctionResultContent from semantic_kernel.filters import FilterTypes +from semantic_kernel.functions import KernelArguments from functions_conversation_memory import EvidenceChunk, EvidenceSource from functions_m365_approvals import M365ApprovalRequired, M365PolicyError -from functions_m365_execution import get_m365_execution_context +from functions_m365_execution import M365ExecutionContext, get_m365_execution_context from m365_interaction import M365_AUTH_INTERACTION_CODES, M365SignInRequired from functions_model_capabilities import ModelTokenBudget from functions_model_budget_runtime import prepare_model_execution_settings @@ -25,6 +26,7 @@ _current_journal = ContextVar("m365_agent_journal", default=None) _current_call_id = ContextVar("m365_agent_call_id", default=None) +_current_step_model = ContextVar("m365_step_model", default=None) _dependencies = {} PAUSED_TOOL_MARKER = "simplechat_m365_tool_waiting" @@ -39,15 +41,98 @@ def configure_m365_agent_continuation( ) -def get_m365_analysis_agent(context): +class M365AnalysisModel: + """The model that runs retained-file analysis batches for one Microsoft 365 request.""" + + def __init__(self, context, *, deployment_name, model_token_budget, service_and_settings): + self.context = context + self.deployment_name = deployment_name + self.model_token_budget = model_token_budget + self._service_and_settings = service_and_settings + + async def service_and_settings(self): + """The chat service and its execution settings; analysis removes their tools.""" + return await self._service_and_settings() + + +def _same_request(bound, context): + return ( + bound.request_id == context.request_id + and bound.data_user_id == context.data_user_id + and bound.conversation_id == context.conversation_id + ) + + +def _agent_analysis_model(journal): + agent = journal.agent + + async def service_and_settings(): + return await agent._get_chat_completion_service_and_settings( + kernel=agent.kernel, arguments=agent.arguments or KernelArguments(), + ) + + return M365AnalysisModel( + journal.context, + deployment_name=getattr(agent, "deployment_name", None), + model_token_budget=getattr(agent, "model_token_budget", None), + service_and_settings=service_and_settings, + ) + + +def get_m365_analysis_model(context): + """The model that runs retained-file analysis for this exact Microsoft 365 request. + + In chat it is the selected agent, through its continuation journal. An orchestration + action step has no agent, so it is the step's own model, bound by + ``m365_step_model_binder``. Neither serves another request, data user or conversation. + """ journal = _current_journal.get() - if journal is None or ( - journal.context.request_id != context.request_id - or journal.context.data_user_id != context.data_user_id - or journal.context.conversation_id != context.conversation_id - ): - raise M365PolicyError("m365_analysis_unavailable", "A selected conversation agent is required for deeper analysis.") - return journal.agent + if journal is not None and _same_request(journal.context, context): + return _agent_analysis_model(journal) + step_model = _current_step_model.get() + if step_model is not None and _same_request(step_model.context, context): + return step_model + raise M365PolicyError("m365_analysis_unavailable", "A selected conversation agent is required for deeper analysis.") + + +def m365_step_model_binder(context, *, service, model_token_budget, tool_schemas=()): + """Validate an orchestration step's own model once, and return its per-call binder. + + A chat agent's continuation journal binds its model's token budget before each tool + call, and offers the agent to retained-file analysis. An orchestration action step + calls its model without an agent, so file reads were refused with + ``model_context_unavailable`` and analysis with ``m365_analysis_unavailable``. The + binder sets both for one function call, for the step's own request only. + """ + setter = _dependencies.get("model_context_setter") + reset = _dependencies.get("model_context_reset") + if setter is None or reset is None: + raise M365PolicyError("m365_continuation_unavailable", "Durable agent continuation is not configured.") + if not isinstance(context, M365ExecutionContext) or not isinstance(model_token_budget, ModelTokenBudget): + raise M365PolicyError("m365_analysis_unavailable", "The step's model is not bound to this Microsoft 365 request.") + schemas = list(tool_schemas) + + async def service_and_settings(): + return service, service.get_prompt_execution_settings_class()(service_id=service.service_id) + + analysis_model = M365AnalysisModel( + context, + deployment_name=getattr(service, "ai_model_id", None), + model_token_budget=model_token_budget, + service_and_settings=service_and_settings, + ) + + @contextmanager + def bind(messages): + budget_token = setter(model_token_budget, list(messages or ()), tool_schemas=schemas) + model_token = _current_step_model.set(analysis_model) + try: + yield + finally: + _current_step_model.reset(model_token) + reset(budget_token) + + return bind def _approval_error(error): diff --git a/application/single_app/functions_m365_analysis_runtime.py b/application/single_app/functions_m365_analysis_runtime.py index e3709bbb7..9e4e2c488 100644 --- a/application/single_app/functions_m365_analysis_runtime.py +++ b/application/single_app/functions_m365_analysis_runtime.py @@ -8,10 +8,9 @@ import re from semantic_kernel.contents import ChatHistory -from semantic_kernel.functions import KernelArguments from conversation_memory_runtime import resolve_m365_memory -from functions_m365_agent_continuation import get_m365_analysis_agent +from functions_m365_agent_continuation import get_m365_analysis_model from functions_m365_analysis_jobs import AnalysisBatchResult, ConversationAnalysisJobRunner from functions_m365_approvals import M365PolicyError, get_m365_approval_service from functions_m365_execution import authorize_m365_publication @@ -63,9 +62,9 @@ async def analyze_m365_memory(context, source, action_id, memory_id, question, a "message": "Use the available excerpts; deeper file analysis was not approved.", "coverage": {"complete": False}, } - agent = get_m365_analysis_agent(context) - model = getattr(agent, "deployment_name", None) - model_budget = resolve_model_token_budget(getattr(agent, "model_token_budget", None) or model) + analysis_model = get_m365_analysis_model(context) + model = analysis_model.deployment_name + model_budget = resolve_model_token_budget(analysis_model.model_token_budget or model) processor_version = "m365-v1-" + hashlib.sha256( json.dumps({"model": model, "budget": asdict(model_budget), "question": question}, sort_keys=True).encode("utf-8") ).hexdigest()[:24] @@ -86,9 +85,7 @@ async def generate(batch): "evidence": batch.evidence, }, ensure_ascii=False) history.add_user_message(content) - service, settings = await agent._get_chat_completion_service_and_settings( - kernel=agent.kernel, arguments=agent.arguments or KernelArguments(), - ) + service, settings = await analysis_model.service_and_settings() try: settings, batch_budget = prepare_model_execution_settings( settings, model_budget, output_limit=1536, diff --git a/application/single_app/functions_orchestration_actions.py b/application/single_app/functions_orchestration_actions.py index 19ce18a40..ac6282c68 100644 --- a/application/single_app/functions_orchestration_actions.py +++ b/application/single_app/functions_orchestration_actions.py @@ -5,8 +5,9 @@ kernel holds only the built-in chart tools, never the action's own functions, so saved visual preferences can be applied there without reaching calls to the integration. -Version: 0.261.238 +Version: 0.261.300 Microsoft 365 actions authorized for their own step in: 0.261.238 +SharePoint and OneDrive steps bind their model's token budget and analysis model in: 0.261.300 """ import asyncio @@ -39,7 +40,10 @@ from functions_orchestration_invocation_capture import require_invocation_capture from functions_orchestration_m365 import ( action_step_scope, + file_step_filter, + file_step_model_limits, is_m365_action_manifest, + m365_file_source, result_refusal, step_error, ) @@ -94,7 +98,12 @@ def _check_access(settings, catalog, action_ref): return selected -def _build_action_model(settings, context, user_id, *, capture_configuration=False): +def _resolve_action_model(settings, context, user_id): + """Resolve and build the step's authorized model once. + + Returns its service, the protocol the service speaks, the resolved endpoint record (None + for the classic deployment settings) and the model context. + """ # Model and plugin dependencies initialize clients; import them only for a running step. from functions_model_endpoint_runtime import ( build_semantic_kernel_chat_service_for_model, @@ -137,9 +146,10 @@ def _build_action_model(settings, context, user_id, *, capture_configuration=Fal deployment, settings, service_id='orchestration-action', model_context=model_context, resolved_model_endpoint=endpoint, ) - if not capture_configuration: - return service + return service, protocol, endpoint, model_context + +def _action_model_configuration(settings, service, protocol, endpoint, model_context): connection = (endpoint or {}).get('connection') or {} if endpoint: configured_endpoint = connection.get('endpoint') @@ -148,7 +158,7 @@ def _build_action_model(settings, context, user_id, *, capture_configuration=Fal prefix = 'azure_apim_gpt' if settings.get('enable_gpt_apim') else 'azure_openai_gpt' configured_endpoint = settings.get(f'{prefix}_endpoint') configured_version = settings.get(f'{prefix}_api_version') - return service, azure_chat_construction_metadata( + return azure_chat_construction_metadata( service, protocol=protocol, provider=(endpoint.get('provider') or 'aoai') if endpoint else 'aoai', configured_endpoint=configured_endpoint, configured_api_version=configured_version, @@ -156,6 +166,67 @@ def _build_action_model(settings, context, user_id, *, capture_configuration=Fal ) +def _build_action_model(settings, context, user_id, *, capture_configuration=False): + service, protocol, endpoint, model_context = _resolve_action_model(settings, context, user_id) + if not capture_configuration: + return service + return service, _action_model_configuration(settings, service, protocol, endpoint, model_context) + + +def _action_model_budget(settings, service, protocol, endpoint, model_context): + """The step model's token budget, from the same records its service was built from. + + Mirrors ``build_agent_model_budget`` with secret-free projections of those records. The + step does not cap its model's output, so no request limit is set: the model's documented + output limit is reserved instead. Raises ``ModelTokenBudgetError`` for invalid limits. + """ + # Budget and endpoint metadata helpers are only needed by a running file step. + from functions_model_capabilities import project_model_budget_metadata, resolve_model_token_budget + from functions_model_endpoint_providers import MODEL_ENDPOINT_PROTOCOL_ANTHROPIC + from functions_model_endpoint_types import get_model_endpoint_api_type, resolve_model_endpoint_request_model + + deployment = getattr(service, 'ai_model_id', None) or model_context.get('model_deployment') + if endpoint: + model_id = str(model_context.get('model_id') or '').strip() + models = [model for model in endpoint.get('models') or [] if isinstance(model, dict)] + record = next( + (model for model in models if model_id and str(model.get('id') or '').strip() == model_id), None, + ) or next( + (model for model in models if resolve_model_endpoint_request_model(endpoint, model) == deployment), + None, + ) + endpoint_metadata = project_model_budget_metadata(endpoint) + provider = endpoint.get('provider') or 'aoai' + if get_model_endpoint_api_type(endpoint) == 'azure_openai': + provider = 'azure' + else: + record = next(( + model for model in (settings.get('gpt_model') or {}).get('selected') or [] + if isinstance(model, dict) and model.get('deploymentName') == deployment + ), None) + endpoint_metadata = {'provider': 'aoai'} + provider = 'aoai' + return resolve_model_token_budget( + project_model_budget_metadata(record) if record else {'deploymentName': deployment}, + endpoint_metadata, provider=provider, + protocol='messages' if protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC else 'chat_completions', + ) + + +def _build_file_action_model(settings, context, user_id, *, capture_configuration=False): + """A SharePoint or OneDrive step's model, its capture configuration, and its token budget. + + File functions bound the content they return by the model's verified token limits, so a + file step resolves the budget from the same single resolution as its service. + """ + service, protocol, endpoint, model_context = _resolve_action_model(settings, context, user_id) + configuration = ( + _action_model_configuration(settings, service, protocol, endpoint, model_context) + if capture_configuration else None + ) + return service, configuration, _action_model_budget(settings, service, protocol, endpoint, model_context) + + def _model_usage(messages): usage = {} seen = set() @@ -450,7 +521,9 @@ async def invoke_action( invocation capture it acquires nothing the capture has not already attested. A Microsoft 365 action runs inside its own step's Microsoft 365 context, keyed by - ``m365_request_key`` so a retry of the step reuses its request and any approval. + ``m365_request_key`` so a retry of the step reuses its request and any approval. A + SharePoint or OneDrive action also binds the step model's token budget around each of + its function calls, and offers that model to deeper file analysis, as a chat agent does. """ # Keep the planner/registry importable without SK and the Azure application bootstrap. from semantic_kernel import Kernel @@ -516,9 +589,13 @@ async def invoke_action( current_settings = deepcopy(current_settings) manifest = deepcopy(manifest) m365_step = is_m365_action_manifest(manifest) + # SharePoint and OneDrive functions also need the step model's token budget. + file_source = m365_file_source(manifest) + m365_context = None + model_budget = None if m365_step: # Sign-in is checked here, before the model or Microsoft Graph is called. - m365_scope.enter_context(action_step_scope( + m365_context = m365_scope.enter_context(action_step_scope( action_ref, user_id=user_id, conversation_id=getattr(context, 'conversation_id', None), request_key=m365_request_key, user_groups=getattr(context, 'active_group_ids', None), origin=m365_origin, @@ -611,9 +688,15 @@ async def stop_after_failure(invocation, next): try: prepared = prepare_action_plugin_manifest(manifest, current_settings) if invocation_capture is not None: - service, model_configuration = _build_action_model( - current_settings, context, user_id, capture_configuration=True, - ) + if file_source: + with file_step_model_limits(file_source): + service, model_configuration, model_budget = _build_file_action_model( + current_settings, context, user_id, capture_configuration=True, + ) + else: + service, model_configuration = _build_action_model( + current_settings, context, user_id, capture_configuration=True, + ) kernel.add_service(service) if model_configuration is None: invocation_capture.refuse() @@ -635,10 +718,25 @@ async def stop_after_failure(invocation, next): kernel.add_filter(FilterTypes.FUNCTION_INVOCATION, guard_function) kernel.add_filter(FilterTypes.AUTO_FUNCTION_INVOCATION, stop_after_failure) if invocation_capture is None: - service = _build_action_model(current_settings, context, user_id) + if file_source: + with file_step_model_limits(file_source): + service, _configuration, model_budget = _build_file_action_model( + current_settings, context, user_id, + ) + else: + service = _build_action_model(current_settings, context, user_id) kernel.add_service(service) if not getattr(service, 'SUPPORTS_FUNCTION_CALLING', False): raise ActionExecutionError('The selected model does not support action functions.') + if file_source: + # Chat binds these through the selected agent; this step's own model stands in. + kernel.add_filter(FilterTypes.AUTO_FUNCTION_INVOCATION, file_step_filter( + m365_context, service=service, model_token_budget=model_budget, source=file_source, + tool_schemas=[ + metadata.model_dump(mode='json', exclude_none=True) + for metadata in kernel.get_full_list_of_function_metadata() + ], + )) system_prompt = ( 'Complete one knowledge-collection step using only the supplied action functions. ' 'Use the action descriptions to choose functions and arguments. You may make ' diff --git a/application/single_app/functions_orchestration_m365.py b/application/single_app/functions_orchestration_m365.py index 4831248fa..cf65a731d 100644 --- a/application/single_app/functions_orchestration_m365.py +++ b/application/single_app/functions_orchestration_m365.py @@ -10,9 +10,14 @@ actions. The helpers here turn Microsoft 365 sign-in, approval and policy refusals into application-owned step failures, instead of findings the model reports as data. -Version: 0.261.270 +SharePoint and OneDrive file functions also need the model's token budget, and deeper file +analysis needs a model. Chat supplies both through the selected agent. An action step has no +agent, so ``file_step_filter`` binds the step's own model around each of its function calls. + +Version: 0.261.300 Implemented in: 0.261.238 Agent steps get their own Microsoft 365 scope in: 0.261.270 +File action steps get their model's token budget and analysis model in: 0.261.300 """ import hashlib @@ -23,15 +28,31 @@ from functions_appinsights import log_event from functions_m365_approvals import M365ApprovalRequired, M365PolicyError -from functions_m365_operations import M365_PLUGIN_TYPES, M365_SOURCES +from functions_m365_operations import M365_ACTION_DEFINITIONS, M365_FILE_SOURCES, M365_PLUGIN_TYPES, M365_SOURCES from functions_msgraph_operations import MSGRAPH_PLUGIN_TYPE from m365_interaction import M365_AUTH_INTERACTION_CODES M365_STEP_ACTION_TYPES = frozenset({*M365_PLUGIN_TYPES, MSGRAPH_PLUGIN_TYPE}) +_FILE_ACTION_SOURCES = { + action_type: definition['source'] for action_type, definition in M365_ACTION_DEFINITIONS.items() + if definition['source'] in M365_FILE_SOURCES +} _APPROVAL_CODES = frozenset({'m365_approval_required', 'm365_approval_pending'}) # Function-result codes that stop a step: no later call can succeed until the user acts. _RESULT_STOP_CODES = frozenset({'execution_context_required', 'source_not_authorized', 'token_acquisition_failed'}) +# The step's model can't bound file content: its token limits are missing or unusable. +_MODEL_LIMIT_CODES = frozenset({ + 'model_context_unavailable', 'model_generation_unbounded', 'model_input_estimate_unavailable', + 'model_context_invalid', 'model_tool_configuration_invalid', +}) +# Retained file evidence couldn't be stored or read safely for this conversation and request. +_EVIDENCE_CODES = frozenset({ + 'memory_unavailable', 'memory_access_denied', 'memory_context_mismatch', 'memory_busy', + 'memory_recovery_required', 'request_memory_binding_required', 'request_memory_mismatch', + 'request_memory_busy', 'invalid_request_memory', 'invalid_model_budget', + 'm365_continuation_unavailable', +}) class OrchestrationM365Error(RuntimeError): @@ -61,6 +82,10 @@ def failure_code(m365_code): return 'm365_sign_in_required' if m365_code in _APPROVAL_CODES: return 'm365_approval_required' + if m365_code in _MODEL_LIMIT_CODES: + return 'm365_model_limits_required' + if m365_code in _EVIDENCE_CODES: + return 'm365_evidence_unavailable' return 'm365_unavailable' @@ -84,6 +109,7 @@ def step_error(error, *, sources=()): def _stops_step(code): return ( code in M365_AUTH_INTERACTION_CODES or code in _APPROVAL_CODES or code in _RESULT_STOP_CODES + or code in _MODEL_LIMIT_CODES or code in _EVIDENCE_CODES or code.startswith('m365_') ) @@ -92,7 +118,9 @@ def result_refusal(value): """The step stop for a Microsoft 365 function result that refused its call, or None. Ordinary Microsoft Graph outcomes, such as nothing found or throttling, stay data that - the model reports. Sign-in, approval and authorization refusals stop the step. + the model reports, as do bounded-coverage limits. Sign-in, approval and authorization + refusals stop the step, and so do model-budget and conversation-evidence failures: they + describe the application, not the user's data. """ if isinstance(value, str): try: @@ -114,6 +142,61 @@ def is_m365_action_manifest(manifest): return isinstance(manifest, dict) and manifest.get('type') in M365_STEP_ACTION_TYPES +def m365_file_source(manifest): + """The file source, ``onedrive`` or ``spo``, of a Microsoft 365 file action, or None.""" + return _FILE_ACTION_SOURCES.get(manifest.get('type')) if isinstance(manifest, dict) else None + + +@contextmanager +def file_step_model_limits(source): + """Stop a file step whose model's token limits can't bound file content. + + The stop keeps the model-budget code for diagnostics and names the step's file source. + The step's Microsoft 365 scope logs it as it leaves. + """ + from functions_model_capabilities import ModelTokenBudgetError + + try: + yield + except ModelTokenBudgetError as error: + raise OrchestrationM365Error( + failure_code(error.code), m365_code=error.code, sources=(source,), + ) from error + + +def file_step_filter(context, *, service, model_token_budget, tool_schemas, source): + """The auto function-invocation filter one SharePoint or OneDrive action step needs. + + File functions bound the content they return by the model's token budget, and deeper + file analysis needs a model. Chat supplies both through the selected agent. An action + step has none, so before 0.261.300 every file read was refused with + ``model_context_unavailable`` and the step still completed. The step's own model must + have verified token limits: without them the step stops here, before its model or + Microsoft Graph is called. + """ + # Continuation loads Semantic Kernel and conversation memory; only a file step needs it. + from functions_m365_agent_continuation import m365_step_model_binder + from functions_model_capabilities import ModelTokenBudget, ModelTokenBudgetError + + with file_step_model_limits(source): + if not isinstance(model_token_budget, ModelTokenBudget): + raise ModelTokenBudgetError('model_context_unavailable', "The step's model has no token budget.") + model_token_budget.remaining_input(0) + try: + binder = m365_step_model_binder( + context, service=service, model_token_budget=model_token_budget, tool_schemas=tool_schemas, + ) + except M365PolicyError as error: + raise step_error(error, sources=(source,)) from error + + async def bind_step_model(invocation, next): + history = getattr(invocation, 'chat_history', None) + with binder(history.messages if history is not None else ()): + await next(invocation) + + return bind_step_model + + def step_request_id(request_key): """A stable request id for one plan step, shared by its retries, so an approval carries over.""" if not isinstance(request_key, str) or not request_key: diff --git a/application/single_app/functions_orchestration_schema.py b/application/single_app/functions_orchestration_schema.py index b959786ad..b2024fd71 100644 --- a/application/single_app/functions_orchestration_schema.py +++ b/application/single_app/functions_orchestration_schema.py @@ -1767,6 +1767,16 @@ def summarize_plan(plan): "Plans can only read Microsoft 365 data. This action's enabled functions all send or change " 'data, so use it from chat instead.' ), + 'm365_model_limits_required': ( + "This step's model has no verified token limits, so SharePoint and OneDrive file content " + "couldn't be used safely. Ask an admin to set this model's token limits in Model Endpoints, " + 'then ask again.' + ), + 'm365_evidence_unavailable': ( + "SharePoint or OneDrive file evidence couldn't be saved or read for this conversation, so " + 'no file content was used. Select Retry from failed step. If it fails again, ask an admin ' + "to check the app's chat storage." + ), 'workflow_runtime_unavailable': ( 'Saved workflows were temporarily unavailable, so this workflow may not have started. ' "Retrying is safe: a workflow this plan already started won't start again." @@ -1863,6 +1873,7 @@ def summarize_plan(plan): M365_STEP_FAILURE_CODES = frozenset({ 'm365_sign_in_required', 'm365_approval_required', 'm365_unavailable', 'm365_shared_conversation', 'm365_read_only', + 'm365_model_limits_required', 'm365_evidence_unavailable', }) # Failures an exception may carry by code, through its ``orchestration_failure_code``. EXCEPTION_FAILURE_CODES = M365_STEP_FAILURE_CODES | {'external_session_required'} diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index 7d17d9842..9e080b2b0 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -445,10 +445,21 @@ limits: analysis, stops and links to Approvals. The user decides there and then selects **Retry from failed step**. A retried step reuses its request, so the decision applies while the action is unchanged. +- **SharePoint and OneDrive use the step's own model.** File searches and reads limit + the excerpts and content they return to what the model can take. Chat measures that + against the selected agent's model. Since **0.261.300**, a plan step measures it + against the model the step runs on, and deeper file analysis runs on that same model + once the user approves it. The model needs published token limits, from the model + catalog or Model Endpoints. Without them, the step stops with "This step's model has + no verified token limits" before it reads anything. Earlier versions refused every + file search and read in a plan with `model_context_unavailable`. The step still + completed, so the answer said the documents couldn't be searched. A refused Microsoft 365 call stops the step instead of becoming findings that the model reports as data. Ordinary Microsoft Graph outcomes, such as nothing found or throttling, -are still findings. An answer that used Microsoft 365 keeps the tool calls in its sources, +are still findings. Since **0.261.300**, file evidence that can't be saved or read for the +conversation also stops the step, with "SharePoint or OneDrive file evidence couldn't be +saved or read". An answer that used Microsoft 365 keeps the tool calls in its sources, so sharing the conversation later asks the user to approve Microsoft 365 history, as it does for a chat answer. Since **0.261.270**, an **Ask an agent** step whose agent uses Microsoft 365 actions gets a step request for that agent's own Microsoft 365 actions, with diff --git a/docs/explanation/fixes/ORCHESTRATION_M365_ACTION_CONTEXT_FIX.md b/docs/explanation/fixes/ORCHESTRATION_M365_ACTION_CONTEXT_FIX.md index a9ab22603..d53e21307 100644 --- a/docs/explanation/fixes/ORCHESTRATION_M365_ACTION_CONTEXT_FIX.md +++ b/docs/explanation/fixes/ORCHESTRATION_M365_ACTION_CONTEXT_FIX.md @@ -217,6 +217,8 @@ limits below describe 0.261.238. ## Related +- [Orchestration Microsoft 365 file action evidence fix](ORCHESTRATION_M365_FILE_ACTION_EVIDENCE_FIX.md), + which gives SharePoint and OneDrive steps their model's token budget in 0.261.300 - [Microsoft 365 actions in plans](../../admin/orchestration.md#microsoft-365-actions-in-plans) - [Microsoft 365 data and approvals](../../guides/microsoft-365-conversation-data.md) - [Orchestration settings document type fix](ORCHESTRATION_SETTINGS_DOCUMENT_TYPE_FIX.md) diff --git a/docs/explanation/fixes/ORCHESTRATION_M365_FILE_ACTION_EVIDENCE_FIX.md b/docs/explanation/fixes/ORCHESTRATION_M365_FILE_ACTION_EVIDENCE_FIX.md new file mode 100644 index 000000000..0e55042c7 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_M365_FILE_ACTION_EVIDENCE_FIX.md @@ -0,0 +1,212 @@ +# Orchestration Microsoft 365 File Action Evidence Fix + +**Version: 0.261.300** + +Fixed in version: **0.261.300**, recorded in +`application/single_app/config.py`. + +This affects the React V2 branch (`paullizer-react-v2-ui`) and deployments built +from it. + +## Issue + +In chat, the user's `m365` agent answered "what is the EVA Swab Tool" from a NASA +PDF in their SharePoint document library. With **Orchestrate** on, the planner +chose a **Use an action** step with the user's **M365 SharePoint** action. The step +made one function call and reported **Completed**, but the answer said the search +"did not return any documents or excerpts because the connected document-evidence +service could not run." + +## Root cause + +### Telemetry + +App Insights for the deployment recorded one Microsoft 365 failure for the +orchestrated step and none for the agent's successful search. App events are +`traces` with the message `[SIMPLE_CHAT_LOG_EVENT]` and the event text in +`customDimensions.sc_message`: + +| Time (UTC) | Event | `sc_failure_code` | `sc_resource` | `sc_operation` | +| --- | --- | --- | --- | --- | +| 2026-10-07 21:40:39 | `[MS_GRAPH_PLUGIN] Microsoft 365 operation could not complete.` | `model_context_unavailable` | `spo` | `search_files` | + +The step's model never sent the search to Microsoft Graph. The function refused the +call before any Microsoft 365 request. + +### Why a plan step had no model budget + +SharePoint and OneDrive file functions bound the excerpts and file content they +return by the model's remaining context. `search_files`, `read_file`, +`read_file_chunk` and `analyze_file` read that room from a task-local budget that +`configure_m365_model_context()` sets. + +- In chat, `LoggingChatCompletionAgent.invoke` runs inside + `m365_agent_continuation`. Its continuation journal binds the agent's + `ModelTokenBudget` before each tool call, and offers the agent to deeper file + analysis. +- An orchestration **Use an action** step calls its model directly, with a bare + Semantic Kernel kernel and no agent. The + [Microsoft 365 action context fix](ORCHESTRATION_M365_ACTION_CONTEXT_FIX.md) in + 0.261.238 gave these steps a Microsoft 365 execution context, but nothing bound a + model budget. + +So every file function refused the call with `model_context_unavailable`: "Select an +agent with verified model token limits before adding file evidence." Calendar and +Email functions don't use the budget, which is why those plan steps worked. + +### Why the step still completed + +`result_refusal()` stopped a step only for sign-in, approval and authorization +refusals. `model_context_unavailable` is none of those, so the refusal reached the +model as an ordinary result. The model reported it as findings, and the final +answer said the documents couldn't be searched. + +### Deeper file analysis could never finish in a plan + +`analyze_m365_memory()` took its model from the chat agent's continuation journal. +In a plan step, `analyze_file` stopped for the user's extended-analysis approval. +After the user approved it and retried the step, the approval was found, but no +agent existed. The step then failed with `m365_unavailable`, telling the user to +check their access, and every retry failed the same way. + +## Changes + +### Each file step binds its own model + +`invoke_action()` in `functions_orchestration_actions.py` treats an +`m365_sharepoint` or `m365_onedrive` action as a file step: + +- `_build_file_action_model()` builds the step's model from the same single + endpoint and model resolution as `_build_action_model()`, which keeps its + signature. `_action_model_budget()` derives the step's `ModelTokenBudget` the way + `build_agent_model_budget()` does for an agent: + - It uses secret-free projections of the resolved model and endpoint records, or + of the classic `gpt_model` deployment. + - It uses the protocol the service speaks. + - It sets no request output limit. The step doesn't cap its model's output, so + the model's documented output limit is reserved. +- `file_step_filter()` in `functions_orchestration_m365.py` is an auto + function-invocation filter. Around each of the step's function calls, it binds: + - The model budget, measured against the step's current chat history and tool + schemas. + - The step's own model, offered to retained-file analysis for this request only. + + Both end with the call. + +### Deeper file analysis uses the step's model + +`get_m365_analysis_model()` in `functions_m365_agent_continuation.py` replaces +`get_m365_analysis_agent()`. It returns the chat agent's model through the +continuation journal, as before. Without a journal, it returns the model that +`m365_step_model_binder()` bound for the same request, data user and conversation. +Any other request is refused with `m365_analysis_unavailable`. +`analyze_m365_memory()` reads the deployment name, budget, service and settings +through this lookup. Batches still remove tools, cap output at 1,536 tokens and +check the model's room. + +### Model limits are checked before the step reads anything + +The step's model must have verified token limits, from the model catalog or Model +Endpoints. `file_step_filter()` checks the budget before the step's model or +Microsoft Graph is called. A model without usable limits stops the step with the +new `m365_model_limits_required` failure, and the stop keeps the budget code, such +as `model_generation_unbounded`. + +### Application failures stop the step + +`result_refusal()` and `failure_code()` now stop a step on two more kinds of +function result. These describe the application, not the user's data: + +| Failure code | Function result codes | +| --- | --- | +| `m365_model_limits_required` | `model_context_unavailable`, `model_generation_unbounded`, `model_input_estimate_unavailable`, `model_context_invalid`, `model_tool_configuration_invalid` | +| `m365_evidence_unavailable` | `memory_unavailable`, `memory_access_denied`, `memory_context_mismatch`, `memory_busy`, `memory_recovery_required`, `request_memory_binding_required`, `request_memory_mismatch`, `request_memory_busy`, `invalid_request_memory`, `invalid_model_budget`, `m365_continuation_unavailable` | + +Bounded-coverage outcomes stay findings that the model reports. These include +`model_context_full`, `memory_hard_limit`, `request_operation_limit`, +`search_limit`, nothing found and throttling. + +Both failures are in `M365_STEP_FAILURE_CODES`, so a step's failure keeps its file +source in `m365_sources`. Their messages name the next step: + +- `m365_model_limits_required`: ask an admin to set the model's token limits in Model + Endpoints, then ask again. A retry runs the plan's approved model, and an Auto plan + whose model limits changed asks for a new plan. +- `m365_evidence_unavailable`: retry, then ask an admin to check the app's chat + storage. + +The V2 run details show both messages. No UI change was needed. + +## Files modified + +| File | Change | +| --- | --- | +| `functions_orchestration_actions.py` | `_resolve_action_model()`, `_action_model_configuration()`, `_build_file_action_model()` and `_action_model_budget()`. `invoke_action()` builds a file step's model with its budget and registers `file_step_filter()`. | +| `functions_orchestration_m365.py` | `m365_file_source()`, `file_step_model_limits()`, `file_step_filter()`, and the model-limit and evidence refusal codes. | +| `functions_m365_agent_continuation.py` | `M365AnalysisModel`, `get_m365_analysis_model()` and `m365_step_model_binder()`. | +| `functions_m365_analysis_runtime.py` | Reads its model through `get_m365_analysis_model()`. | +| `functions_orchestration_schema.py` | `m365_model_limits_required` and `m365_evidence_unavailable`. | +| `config.py` | Version `0.261.300`. | + +## Validation + +### Tests + +`functional_tests/test_orchestration_m365_file_actions.py` runs: + +- The real action runner, step scope, Microsoft 365 runtime, approval service and + SharePoint plugin. +- The production file-runtime wiring, through `configure_m365_file_runtime()`, + through a real Flask bridge. + +Microsoft Graph, blob storage, Cosmos and the model are doubled, and network access +is blocked. The tests check that: + +- A SharePoint step's `search_files` finds the document through Microsoft Graph + search, and returns its excerpt with a measured context budget. The step + completes with the excerpt in its findings. +- A step model without verified token limits stops the step with + `m365_model_limits_required`, before the model or Microsoft Graph is called. +- Unavailable conversation evidence fails the step with + `m365_evidence_unavailable`, instead of findings. +- `analyze_file` stops for the user's approval. After the approval, a retry reuses + the captured file, and the analysis batch runs on the step's own model. The batch + carries the file's evidence, has no tools and caps output at 1,536 tokens. +- The binding serves only its own request, data user and conversation, and ends + with the call. +- Model-limit and evidence codes stop the step, and capacity outcomes stay findings. +- The step budget comes from the resolved endpoint record, the classic deployment + settings, or the Anthropic protocol. An unlisted model has no usable budget. + +Four deliberate breaks each make at least one test fail: + +- Not registering the step filter, which is the shipped bug. +- Not binding the analysis model. +- Keeping evidence refusals as findings. +- Skipping the up-front limits check. + +`functional_tests/test_m365_runtime_adapters.py` now supplies the analysis model +through `get_m365_analysis_model()`. The existing orchestration, Microsoft 365 +provider, file analysis and continuation tests pass unchanged. + +### Before and after + +| Before | After | +| --- | --- | +| Every SharePoint or OneDrive file function in a plan refused the call with `model_context_unavailable`. | The step's own model bounds file content, as the agent's model does in chat. | +| The refusal became findings, the step completed, and the answer said the documents couldn't be searched. | Model-limit and evidence failures stop the step with a message that names the fix. | +| After the user approved deeper analysis, every retry failed with `m365_unavailable`. | A retry after the approval analyzes the file on the step's own model. | + +## Limitations + +- Each `analyze_file` call processes one captured chunk. **Max auto-invoke + attempts** limits a step's function calls, so a long file can finish with partial + coverage. The analysis checkpoint lets a retry continue where it stopped. +- A model without published token limits can't add file evidence in a plan, the + same as in chat. Set the limits in Model Endpoints, or choose a catalog model. + +## Related + +- [Orchestration Microsoft 365 action context fix](ORCHESTRATION_M365_ACTION_CONTEXT_FIX.md) +- [Microsoft 365 actions in plans](../../admin/orchestration.md#microsoft-365-actions-in-plans) +- [Logging tags](../../reference/logging-tags.md) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 6527230b5..284a1b395 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,26 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.300)** + +#### Bug Fixes + +* **Orchestrated SharePoint and OneDrive Actions Find Documents Again** + * With Orchestrate on, a **Use an action** step with a Microsoft 365 SharePoint or OneDrive action never searched or read any file. Every file function refused the call with `model_context_unavailable` before reaching Microsoft Graph, because the step had no model token budget. Chat binds that budget through the selected agent, and a plan step has no agent. The step still reported completed, so the answer said the documents couldn't be searched, while the same agent in chat found them. Production telemetry for the reported run showed one `search_files` refusal for `spo` with that code. + * A file step now measures what it reads against the model it runs on, the same way chat does with the agent's model. The model's token limits come from the model catalog or Model Endpoints. + * (Ref: `functions_orchestration_actions.py` `_build_file_action_model`, `_action_model_budget`, `functions_orchestration_m365.py` `file_step_filter`, [Orchestration M365 File Action Evidence Fix](fixes/ORCHESTRATION_M365_FILE_ACTION_EVIDENCE_FIX.md)) + +* **Deeper File Analysis Finishes in a Plan After Approval** + * In a plan, `analyze_file` stopped for the user's extended-analysis approval. After the user approved and retried, it failed with "Microsoft 365 access for this step could not be confirmed", because analysis took its model only from a chat agent. Every retry failed the same way. + * Analysis now runs on the step's own model, for that step's request only. A retry after the approval reuses the captured file and completes. + * (Ref: `functions_m365_agent_continuation.py` `get_m365_analysis_model`, `m365_step_model_binder`, `functions_m365_analysis_runtime.py`) + +* **File Steps Fail Clearly Instead of Reporting Missing Documents** + * A SharePoint or OneDrive step whose model has no verified token limits now stops before reading anything, with the new `m365_model_limits_required` failure. Its message says to have an admin set the model's limits in Model Endpoints, then ask again. + * File evidence that can't be saved or read for the conversation now stops the step with the new `m365_evidence_unavailable` failure. Previously both reached the model as findings, so the step completed with a misleading answer. + * Bounded-coverage outcomes, such as a full context window or a request's file limits, are still reported as findings. + * (Ref: `functions_orchestration_m365.py` `result_refusal`, `failure_code`, `functions_orchestration_schema.py` `FAILURE_MESSAGES`, `M365_STEP_FAILURE_CODES`) + ### **(v0.261.299)** #### New Features diff --git a/docs/reference/logging-tags.md b/docs/reference/logging-tags.md index 99bacd9db..e260e43c0 100644 --- a/docs/reference/logging-tags.md +++ b/docs/reference/logging-tags.md @@ -401,7 +401,7 @@ file names, file content, prompts, or model responses. | `[ORCHESTRATION] Retrying the answer model without a JSON response format.` | Information | `sc_reason` (`json_format_retry`), `sc_stage`, `sc_error_type`. Content preparation asks the endpoint for a JSON object; an endpoint that refuses that option is asked again without it. | | `[ORCHESTRATION_ADAPTERS] Native result bridge did not complete.` | Warning | `sc_stage`, `sc_failure_code`, `sc_execution_code`, such as `native_compute_source_identity_mismatch` when a saved spreadsheet location now points at a different revision. | | `[CONTENT_SCREENING] Current source authority could not be verified.` | Warning | `sc_failure_code`, `sc_authority_reason`, `sc_error_type`. | -| `[ORCHESTRATION_M365] A Microsoft 365 step stopped.` | Warning | Since **0.261.238**. `sc_failure_code` (`m365_sign_in_required`, `m365_approval_required`, `m365_unavailable`, `m365_shared_conversation`, `m365_read_only` or `external_session_required`), `sc_authority_reason` (the Microsoft 365 refusal code, such as `interactive_auth_required`, `authentication_required`, `m365_action_not_authorized` or `m365_shared_conversation_unsupported`), `sc_capability_id`, and `sc_source_count`. | +| `[ORCHESTRATION_M365] A Microsoft 365 step stopped.` | Warning | Since **0.261.238**. `sc_failure_code` (`m365_sign_in_required`, `m365_approval_required`, `m365_unavailable`, `m365_shared_conversation`, `m365_read_only` or `external_session_required`), `sc_authority_reason` (the Microsoft 365 refusal code, such as `interactive_auth_required`, `authentication_required`, `m365_action_not_authorized` or `m365_shared_conversation_unsupported`), `sc_capability_id`, and `sc_source_count`. Since **0.261.300**, a SharePoint or OneDrive step can also stop with `m365_model_limits_required`, with an authority reason such as `model_generation_unbounded` or `model_context_unavailable`, or `m365_evidence_unavailable`, with an authority reason such as `memory_unavailable`. | | `[MS_GRAPH_PLUGIN] Microsoft 365 operation could not complete.` | Warning | `sc_operation`. Since **0.261.238**, also `sc_failure_code`, the refusal code (for example `m365_context_required`), and `sc_resource`, the source (`email`, `calendar`, `onedrive` or `spo`). Earlier versions logged only the code's length, `sc_error_code_length`. | Every `[ORCHESTRATION_EXECUTOR]`, `[ORCHESTRATION_ADAPTERS]` and diff --git a/functional_tests/test_m365_runtime_adapters.py b/functional_tests/test_m365_runtime_adapters.py index 9d24c6553..a8db65cdc 100644 --- a/functional_tests/test_m365_runtime_adapters.py +++ b/functional_tests/test_m365_runtime_adapters.py @@ -1,8 +1,9 @@ # test_m365_runtime_adapters.py """ Integration tests for staged file processing and subject-owned resume adapters. -Version: 0.261.035 +Version: 0.261.300 Implemented in: 0.261.029 +Analysis reads its model through get_m365_analysis_model in: 0.261.300 Real memory manifests and batch execution are used with mocked external model I/O. Resume tests prove that authenticated decisions enqueue only their own job. @@ -248,18 +249,17 @@ async def get_chat_message_contents(self, *, chat_history, settings): assert settings.tools is None return [types.SimpleNamespace(content=f"Findings from batch {len(model_calls)}")] - class Agent: + class AnalysisModel: deployment_name = "gpt-5.6-terra" - kernel = object() - arguments = None + model_token_budget = None - async def _get_chat_completion_service_and_settings(self, **kwargs): + async def service_and_settings(self): return Model(), OpenAIChatPromptExecutionSettings( tools=[{"type": "function", "function": {"name": "must_not_be_called", "parameters": {"type": "object", "properties": {}}}}], max_completion_tokens=10000, ) - runtime.get_m365_analysis_agent = lambda context: Agent() + runtime.get_m365_analysis_model = lambda context: AnalysisModel() runtime.get_m365_approval_service = lambda: types.SimpleNamespace( authorize_extended_analysis=lambda *args: {"mode": "extended"}, ) diff --git a/functional_tests/test_orchestration_m365_file_actions.py b/functional_tests/test_orchestration_m365_file_actions.py new file mode 100644 index 000000000..608a3ae72 --- /dev/null +++ b/functional_tests/test_orchestration_m365_file_actions.py @@ -0,0 +1,481 @@ +# test_orchestration_m365_file_actions.py +#!/usr/bin/env python3 +""" +Functional test for SharePoint and OneDrive actions in chat orchestration. +Version: 0.261.300 +Implemented in: 0.261.300 + +An orchestration "Use an action" step calls its model without a chat agent. SharePoint and +OneDrive file functions bound the content they return by the model's token budget, which +chat binds through the selected agent's continuation journal, and deeper file analysis needs +that agent's model. A plan step had neither, so ``search_files`` returned +``model_context_unavailable``. The step still reported completed, and the answer said the +documents couldn't be searched, while the same agent in chat found them. + +These tests run the real action runner, step scope, Microsoft 365 runtime, approval service, +SharePoint plugin and the production file-runtime wiring (``configure_m365_file_runtime``) +through a real Flask bridge. Microsoft Graph, blob storage, Cosmos and the model are doubled, +and network access is blocked. +""" + +import ast +import importlib +import json +import re +import sys +from copy import deepcopy +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import urlsplit + +import pytest +import requests +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.contents import AuthorRole, ChatMessageContent, FunctionCallContent, FunctionResultContent + +TESTS = Path(__file__).resolve().parent +sys.path.insert(0, str(TESTS)) + +# The shared harness sets the application path and doubles; its fixtures run here too. +from test_orchestration_m365_actions import ( # noqa: E402,F401 + ACTION_REF, CONVERSATION, TENANT, USER, env, module, request_record, run_step, world, +) +from test_support.m365 import CosmosContainer # noqa: E402 + +FILE_TEXT = ( + "The EVA Swab Tool collects microbial samples from spacesuits and spacecraft surfaces " + "during spacewalks, supporting planetary protection and astrobiology research." +) +WEB_URL = "https://contoso.sharepoint.com/sites/research/Documents/eva-swab-tool.txt" +SNIPPET = "The EVA Swab Tool collects microbial samples during spacewalks." +ANALYSIS = "The EVA Swab Tool samples spacesuit and spacecraft surfaces for microbes." +ENDPOINT = { + "id": "endpoint-1", "provider": "aoai", + "connection": {"endpoint": "https://example.invalid", "openai_api_version": "2024-10-21"}, + "models": [{"id": "model-1", "deploymentName": "test-model", "modelName": "gpt-5.6-terra"}], +} +MODEL_CONTEXT = {"endpoint_id": "endpoint-1", "model_id": "model-1", "provider": "aoai"} + + +def sharepoint_action(): + return { + "id": "sharepoint", "name": "m365_sharepoint", "display_name": "SharePoint", + "type": "m365_sharepoint", "description": "Search SharePoint document libraries.", + "action_ref": ACTION_REF, "scope_type": "personal", "scope_id": USER, "user_id": USER, + "is_enabled": True, "additionalFields": {"m365_capabilities": {}}, + } + + +class GraphResponse: + def __init__(self, payload=None, status=200, *, body=None, headers=None): + self.status_code = status + self.body = json.dumps(payload).encode() if body is None else body + self.headers = headers or {"Content-Type": "application/json"} + + def iter_content(self, chunk_size): + for index in range(0, len(self.body), chunk_size): + yield self.body[index:index + chunk_size] + + def close(self): + pass + + +def file_item(): + return { + "id": "item-1", "name": "eva-swab-tool.txt", "size": len(FILE_TEXT.encode()), + "file": {"mimeType": "text/plain"}, "parentReference": {"driveId": "drive-1", "siteId": "site-1"}, + "webUrl": WEB_URL, "eTag": '"version-1"', "cTag": '"content-1"', + "lastModifiedDateTime": "2026-09-17T12:00:00Z", + } + + +def graph(state): + def request(method, url, **kwargs): + path = urlsplit(url).path.removeprefix("/v1.0") + state.graph_calls.append({"method": method, "path": path}) + if path == "/me": + # No Copilot license: search uses Microsoft Graph search, as for most users. + return GraphResponse({"id": USER, "assignedLicenses": [], "assignedPlans": []}) + if path == "/search/query": + return GraphResponse({"value": [{"hitsContainers": [{ + "total": 1, "moreResultsAvailable": False, + "hits": [{"resource": file_item(), "summary": SNIPPET}], + }]}]}) + if path == "/drives/drive-1": + return GraphResponse({ + "id": "drive-1", "driveType": "documentLibrary", + "webUrl": "https://contoso.sharepoint.com/sites/research/Documents", + }) + if path == "/drives/drive-1/items/item-1": + return GraphResponse(file_item()) + if path == "/drives/drive-1/items/item-1/content": + body = FILE_TEXT.encode() + return GraphResponse(body=body, headers={"Content-Type": "text/plain", "Content-Length": str(len(body))}) + raise AssertionError(f"Unexpected Microsoft Graph request: {method} {path}") + + return request + + +class MemoryBlobs: + """Blob storage for conversation memory, held in memory with real ETag semantics.""" + + def __init__(self, storage): + self.storage = storage + self.records = {} + self.generation = 0 + + def read(self, container, name, *, max_bytes): + record = self.records.get((container, name)) + if record is None: + raise self.storage.MemoryNotFoundError("No test blob.") + if len(record.data) > max_bytes: + raise AssertionError("The test attempted an oversized blob read.") + return record + + def put(self, container, name, data, *, etag=None): + current = self.records.get((container, name)) + if etag is None and current is not None or etag is not None and (current is None or current.etag != etag): + raise self.storage.MemoryConflictError("Test ETag conflict.") + self.generation += 1 + record = self.storage.BlobRecord(bytes(data), f"etag-{self.generation}") + self.records[(container, name)] = record + return record.etag + + def delete(self, container, name, *, etag): + current = self.records.get((container, name)) + if current is None or current.etag != etag: + raise self.storage.MemoryConflictError("Test ETag conflict.") + del self.records[(container, name)] + + +def call(function, call_id, **arguments): + return ChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[FunctionCallContent(id=call_id, name=f"m365_sharepoint-{function}", arguments=json.dumps(arguments))], + ) + + +def reply(text): + return ChatMessageContent(role=AuthorRole.ASSISTANT, content=text) + + +def as_result(value): + """A function result as the step's model received it, as a dictionary.""" + if isinstance(value, dict): + return value + try: + return json.loads(value) + except (TypeError, ValueError): + return ast.literal_eval(str(value)) + + +def prepared_memory_id(history): + """The memory ID the step's earlier prepare_file call returned, read from its own history.""" + for message in reversed(history.messages): + for item in message.items: + if isinstance(item, FunctionResultContent) and item.function_name == "prepare_file": + return as_result(item.result)["memory_id"] + raise AssertionError("prepare_file has not returned a memory ID") + + +@pytest.fixture(scope="module") +def file_env(env): + """The production file-runtime wiring, with conversation memory supplied per test.""" + holder = SimpleNamespace(binding=None) + sys.modules["conversation_memory_runtime"] = module( + "conversation_memory_runtime", resolve_m365_memory=lambda context: holder.binding(context), + ) + # Plain-text extraction defers to the config-dependent content helpers; read the file directly. + sys.modules["functions_content"] = module( + "functions_content", extract_text_file=lambda path: Path(path).read_text(encoding="utf-8"), + ) + file_runtime = importlib.import_module("functions_m365_file_runtime") + file_runtime.configure_m365_file_runtime() + yield SimpleNamespace( + holder=holder, file_runtime=file_runtime, + retrieval=importlib.import_module("functions_m365_retrieval"), + continuation=importlib.import_module("functions_m365_agent_continuation"), + sharepoint=importlib.import_module("semantic_kernel_plugins.m365_sharepoint_plugin"), + memory=importlib.import_module("functions_conversation_memory"), + storage=importlib.import_module("conversation_memory_storage"), + capabilities=importlib.import_module("functions_model_capabilities"), + transport=importlib.import_module("functions_m365_transport"), + ) + + +@pytest.fixture +def file_world(env, world, file_env, monkeypatch): + blobs = MemoryBlobs(file_env.storage) + + def authorize(context, operation): + return ( + context.tenant_id == TENANT and context.conversation_id == CONVERSATION + and context.principal_id == USER and context.storage_owner == USER + ) + + def publish(context, run, grant_context): + raise AssertionError("A private conversation never publishes file evidence.") + + store = file_env.memory.ConversationMemoryStore( + transport=blobs, authorize_access=authorize, authorize_publish=publish, + log_event=lambda *args, **kwargs: None, + ) + file_env.holder.binding = lambda context: (store, file_env.memory.MemoryContext( + tenant_id=context.tenant_id, principal_id=context.data_user_id, + conversation_id=context.conversation_id, storage_owner=USER, request_id=context.request_id, + )) + monkeypatch.setattr(file_env.file_runtime, "cosmos_m365_execution_runs_container", world.jobs) + world.action = sharepoint_action() + world.store = store + world.script = [] + world.model_calls = [] + # Each function result, as the step's model last received it in its chat history. + world.results = {} + world.endpoint = deepcopy(ENDPOINT) + + class Loader: + def __init__(self, kernel): + self.kernel = kernel + self.plugin_instances = [] + + def load_plugin_from_manifest(self, manifest, user_id): + plugin = file_env.sharepoint.M365SharePointPlugin(deepcopy(manifest)) + world.loaded_functions.append(sorted(plugin.get_functions())) + self.plugin_instances.append(plugin) + self.kernel.add_plugin(plugin.get_kernel_plugin("m365_sharepoint")) + return True + + async def model(service, history, settings): + world.model_calls.append({"history": str(history), "settings": settings}) + for message in history.messages: + for item in message.items: + if isinstance(item, FunctionResultContent): + world.results[item.function_name] = as_result(item.result) + if not world.script: + raise AssertionError("The model was called more often than the test scripted") + return [world.script.pop(0)(history)] + + world.loader = Loader + monkeypatch.setattr(requests, "request", graph(world)) + monkeypatch.setattr(AzureChatCompletion, "_inner_get_chat_message_contents", model) + # The step's model resolves the way production does; only endpoint lookup and client + # construction are doubled, so the budget comes from the real resolution helpers. + monkeypatch.setattr(env.actions, "_resolve_action_model", lambda settings, context, user_id: ( + world.service, "azure_openai", deepcopy(world.endpoint), dict(MODEL_CONTEXT), + )) + monkeypatch.setattr(env.actions, "_action_model_configuration", lambda *args: {"provider": "aoai", "parameters": {}}) + yield world + file_env.holder.binding = None + + +def stopped(env, world): + with pytest.raises(env.orchestration.OrchestrationM365Error) as caught: + run_step(env, world) + return caught.value + + +def approvals_service(env, monkeypatch): + approvals = CosmosContainer() + service = env.approvals.M365ApprovalService( + container_factory=lambda: approvals, notification_sender=lambda approval: None, + decision_validator=lambda approval: True, + ) + monkeypatch.setattr(env.approvals, "_service", service) + return service + + +def test_sharepoint_search_step_reads_documents_with_the_step_models_budget(env, file_env, file_world): + file_world.script = [ + lambda history: call("search_files", "call-1", query="EVA Swab Tool"), + lambda history: reply("SharePoint: the EVA Swab Tool collects microbial samples during spacewalks."), + ] + result, _context = run_step(env, file_world) + + assert result["calls"] == 1 + assert "EVA Swab Tool" in result["findings"] + found = file_world.results["search_files"] + assert found["status"] == "ok" and found["source"] == "spo" + assert [item["display_name"] for item in found["results"]] == ["eva-swab-tool.txt"] + assert "EVA Swab Tool collects microbial samples" in found["results"][0]["excerpts"][0]["text"] + assert found["coverage"]["context_tokens"] > 0 + assert [call_info["path"] for call_info in file_world.graph_calls] == [ + "/me", "/search/query", "/drives/drive-1/items/item-1", "/drives/drive-1", + ] + # The excerpt reached the model as data, not a refusal it could describe as findings. + assert "EVA Swab Tool collects microbial samples" in file_world.model_calls[1]["history"] + assert not any("model_context_unavailable" in json.dumps(extra, default=str) for _message, extra in file_world.logs) + record = request_record(env, file_world) + assert record["status"] == "completed" and record["memory_budget_run_id"] + + +def test_step_model_without_verified_limits_stops_before_the_model_or_graph(env, file_env, file_world): + file_world.endpoint["models"][0]["modelName"] = "unlisted-model" + error = stopped(env, file_world) + + assert error.orchestration_failure_code == "m365_model_limits_required" + assert error.m365_code == "model_generation_unbounded" + assert error.m365_sources == ("spo",) + assert file_world.model_calls == [] and file_world.graph_calls == [] + assert request_record(env, file_world)["status"] == "failed" + failure = env.schema.failure_from_exception(error) + assert failure == { + "code": "m365_model_limits_required", + "message": env.schema.FAILURE_MESSAGES["m365_model_limits_required"], + "m365_sources": ["spo"], + } + stop_logs = [extra for message, extra in file_world.logs if "A Microsoft 365 step stopped" in message] + assert stop_logs == [{ + "failure_code": "m365_model_limits_required", "authority_reason": "model_generation_unbounded", + "capability_id": "action_invoke", "source_count": 1, + }] + + +def test_unavailable_conversation_evidence_fails_the_step_instead_of_becoming_findings(env, file_env, file_world): + def unavailable(context): + raise file_env.storage.MemoryUnavailableError("Chat storage is not configured.") + + file_env.holder.binding = unavailable + file_world.script = [ + lambda history: call("search_files", "call-1", query="EVA Swab Tool"), + lambda history: reply("The document-evidence service could not run."), + ] + error = stopped(env, file_world) + + assert error.orchestration_failure_code == "m365_evidence_unavailable" + assert error.m365_code == "memory_unavailable" + assert error.m365_sources == ("spo",) + # The refusal stopped the step before the model could report it as findings. + assert len(file_world.model_calls) == 1 + assert request_record(env, file_world)["status"] == "failed" + + +def test_analysis_after_approval_runs_with_the_step_model(env, file_env, file_world, monkeypatch): + service = approvals_service(env, monkeypatch) + question = "What is the EVA Swab Tool used for?" + + def analyze(history): + return call("analyze_file", "call-2", memory_id=prepared_memory_id(history), question=question) + + file_world.script = [ + lambda history: call("prepare_file", "call-1", drive_id="drive-1", item_id="item-1"), + analyze, + ] + error = stopped(env, file_world) + + assert error.orchestration_failure_code == "m365_approval_required" + approval = service.get_approval(error.approval_id, USER) + assert approval["request_type"] == env.approvals.TYPE_EXTENDED_ANALYSIS and approval["status"] == "pending" + record = request_record(env, file_world) + assert record["status"] == "failed" and record["approval_id"] == error.approval_id + downloads = [item for item in file_world.graph_calls if item["path"].endswith("/content")] + assert len(downloads) == 1 + + service.decide(error.approval_id, USER, {"choice": "request"}) + file_world.model_calls.clear() + file_world.script = [ + lambda history: call("prepare_file", "call-1", drive_id="drive-1", item_id="item-1"), + analyze, + lambda history: reply(ANALYSIS), + lambda history: reply(f"SharePoint analysis: {ANALYSIS}"), + ] + result, _context = run_step(env, file_world) + + assert result["calls"] == 2 + analysis = file_world.results["analyze_file"] + assert analysis["status"] == "completed" and analysis["findings"] == ANALYSIS + assert re.fullmatch(r"[0-9a-f]{32}", analysis["analysis_id"]) + # The retry reused the captured file, and the analysis batch ran on the step's own model, + # with its evidence, no tools and the bounded batch output. + assert len([item for item in file_world.graph_calls if item["path"].endswith("/content")]) == 1 + batch = next(item for item in file_world.model_calls if item["settings"].function_choice_behavior is None) + assert FILE_TEXT in batch["history"] and question in batch["history"] + assert not getattr(batch["settings"], "tools", None) + assert 1536 in (getattr(batch["settings"], "max_completion_tokens", None), getattr(batch["settings"], "max_tokens", None)) + retried = request_record(env, file_world) + assert retried["status"] == "completed" and retried["approval_id"] == error.approval_id + + +def test_step_binding_serves_only_its_own_request_and_ends_with_the_call(env, file_env, file_world): + context = env.execution.M365ExecutionContext( + actor_user_id=USER, data_user_id=USER, tenant_id=TENANT, + conversation_id=CONVERSATION, request_id="orch-request-1", + ) + budget = file_env.capabilities.resolve_model_token_budget( + {"modelName": "gpt-5.6-terra"}, {"provider": "aoai"}, provider="aoai", + ) + bind = file_env.continuation.m365_step_model_binder( + context, service=file_world.service, model_token_budget=budget, tool_schemas=[], + ) + with pytest.raises(file_env.transport.M365ProviderError) as unbound: + file_env.file_runtime.resolve_m365_model_room(context) + assert unbound.value.code == "model_context_unavailable" + + with bind([ChatMessageContent(role=AuthorRole.USER, content="What is the EVA Swab Tool?")]): + room = file_env.file_runtime.resolve_m365_model_room(context) + assert 0 < room < budget.context_window + model = file_env.continuation.get_m365_analysis_model(context) + assert model.model_token_budget is budget and model.deployment_name == "test-model" + for foreign in ( + replace(context, request_id="orch-request-2"), + replace(context, actor_user_id="user-2", data_user_id="user-2"), + replace(context, conversation_id="conversation-2"), + ): + with pytest.raises(env.approvals.M365PolicyError) as refused: + file_env.continuation.get_m365_analysis_model(foreign) + assert refused.value.code == "m365_analysis_unavailable" + + with pytest.raises(file_env.transport.M365ProviderError): + file_env.file_runtime.resolve_m365_model_room(context) + with pytest.raises(env.approvals.M365PolicyError): + file_env.continuation.get_m365_analysis_model(context) + + +def test_file_refusals_and_capacity_outcomes_map_to_the_right_step_result(env, file_env, file_world): + refusal = env.orchestration.result_refusal + for code in ("model_context_full", "memory_hard_limit", "request_operation_limit", "search_limit", "not_found"): + assert refusal({"error": {"code": code}, "source": "spo"}) is None + for code in ( + "model_context_unavailable", "model_generation_unbounded", "model_input_estimate_unavailable", + "model_context_invalid", "model_tool_configuration_invalid", + ): + stop = refusal({"error": {"code": code}, "source": "spo"}) + assert stop.orchestration_failure_code == "m365_model_limits_required" and stop.m365_sources == ("spo",) + for code in ( + "memory_unavailable", "memory_access_denied", "memory_context_mismatch", "memory_busy", + "memory_recovery_required", "request_memory_binding_required", "request_memory_mismatch", + "request_memory_busy", "invalid_request_memory", "invalid_model_budget", + "m365_continuation_unavailable", + ): + stop = refusal(json.dumps({"error": {"code": code}, "source": "onedrive"})) + assert stop.orchestration_failure_code == "m365_evidence_unavailable" and stop.m365_sources == ("onedrive",) + assert env.orchestration.m365_file_source({"type": "m365_sharepoint"}) == "spo" + assert env.orchestration.m365_file_source({"type": "m365_onedrive"}) == "onedrive" + assert env.orchestration.m365_file_source({"type": "m365_email"}) is None + + +def test_step_budget_comes_from_the_resolved_model_records(env, file_env, file_world): + budget_for = env.actions._action_model_budget + service = file_world.service + endpoint_budget = budget_for({}, service, "azure_openai", deepcopy(ENDPOINT), dict(MODEL_CONTEXT)) + assert endpoint_budget.model_id == "gpt-5.6-terra" and endpoint_budget.provider == "azure" + assert endpoint_budget.request_output_limit is None and endpoint_budget.remaining_input(0) > 0 + + legacy = budget_for( + {"gpt_model": {"selected": [{"deploymentName": "test-model", "modelName": "gpt-5.6-terra"}]}}, + service, "azure_openai", None, {}, + ) + assert legacy.model_id == "gpt-5.6-terra" and legacy.remaining_input(0) > 0 + + messages = budget_for({}, service, "anthropic", deepcopy(ENDPOINT), dict(MODEL_CONTEXT)) + assert messages.protocol == "messages" + + unlisted = deepcopy(ENDPOINT) + unlisted["models"][0]["modelName"] = "unlisted-model" + unverified = budget_for({}, service, "azure_openai", unlisted, dict(MODEL_CONTEXT)) + with pytest.raises(file_env.capabilities.ModelTokenBudgetError): + unverified.remaining_input(0) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-q"]))