Complete resilient Data Management migrations

application/single_app/app.py

@@ -286,7 +286,7 @@ def start_background_tasks():
             print("Background tasks disabled for this web process.")
             _background_tasks_started = True
             return
-        start_background_task_threads()
+        start_background_task_threads(app=app)
         _background_tasks_started = True
 
 

application/single_app/background_tasks.py

@@ -684,14 +684,14 @@ def run_tabular_generated_output_scheduler_loop():
         time.sleep(30)
 
 
-def run_data_management_scheduler_loop():
-    """Queue due Data Management backup jobs across scaled-out workers."""
+def run_data_management_scheduler_loop(app=None):
+    """Queue due backup and recoverable migration jobs across scaled-out workers."""
     while True:
         lock_document = None
         try:
             lock_document = acquire_distributed_task_lock('data_management_scheduler_scan', lease_seconds=300)
             if lock_document:
-                check_due_data_management_jobs_once()
+                check_due_data_management_jobs_once(app=app)
         except Exception as exc:
             print(f"Error in Data Management scheduler check: {exc}")
             log_event(f"[DataManagement] Error in scheduler check: {exc}", level=logging.ERROR)
@@ -736,7 +736,7 @@ def run_app_maintenance_loop():
         time.sleep(max(int(sleep_seconds or 3600), 15))
 
 
-def start_background_task_threads():
+def start_background_task_threads(app=None):
     """Start all background task loops for the current process."""
     task_specs = [
         ('Logging timer background task started.', run_logging_timer_loop),
@@ -747,7 +747,7 @@ def start_background_task_threads():
         ('Workflow scheduler background task started.', run_workflow_scheduler_loop),
         ('File Sync scheduler background task started.', run_file_sync_scheduler_loop),
         ('Tabular generated-output scheduler background task started.', run_tabular_generated_output_scheduler_loop),
-        ('Data Management scheduler background task started.', run_data_management_scheduler_loop),
+        ('Data Management scheduler background task started.', lambda: run_data_management_scheduler_loop(app=app)),
         ('App maintenance background task started.', run_app_maintenance_loop),
     ]
 

application/single_app/config.py

@@ -97,7 +97,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.250.059"
+VERSION = "0.250.078"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')

application/single_app/foundry_agent_runtime.py

@@ -42,7 +42,13 @@
 FOUNDRY_INTERNAL_METADATA_KEYS = {
     "active_group_ids",
     "active_public_workspace_ids",
+    "document_context_requested",
+    "document_scope",
+    "group_id",
+    "selection_mode",
+    "selected_document_id",
     "selected_document_ids",
+    "user_id",
 }
 FOUNDRY_FILE_SEARCHABLE_CONTEXT_MAX_CHARS = 6000
 FOUNDRY_FILE_SEARCHABLE_CONTEXT_HEADER = "Attached file searchable summary"
@@ -391,6 +397,14 @@ async def execute_foundry_agent(
 ) -> FoundryAgentInvocationResult:
     """Invoke a Foundry agent using Semantic Kernel's AzureAIAgent abstraction."""
 
+    message_history = _filter_foundry_document_context_messages(
+        message_history,
+        include_document_context=_coerce_bool(
+            foundry_settings.get("include_document_context"),
+            True,
+        ),
+    )
+
     agent_id = (foundry_settings.get("agent_id") or "").strip()
     if not agent_id:
         raise FoundryAgentInvocationError(
@@ -491,6 +505,14 @@ async def execute_new_foundry_agent(
 ) -> FoundryAgentInvocationResult:
     """Invoke the new Foundry application runtime through its Responses protocol endpoint."""
 
+    message_history = _filter_foundry_document_context_messages(
+        message_history,
+        include_document_context=_coerce_bool(
+            foundry_settings.get("include_document_context"),
+            True,
+        ),
+    )
+
     application_name = _resolve_new_foundry_application_name(foundry_settings)
     endpoint = _resolve_endpoint(foundry_settings, global_settings)
     responses_api_version = (
@@ -573,6 +595,14 @@ async def execute_new_foundry_agent_stream(
 ) -> AsyncIterator[FoundryAgentStreamMessage]:
     """Stream a new Foundry application response through the Responses API."""
 
+    message_history = _filter_foundry_document_context_messages(
+        message_history,
+        include_document_context=_coerce_bool(
+            foundry_settings.get("include_document_context"),
+            True,
+        ),
+    )
+
     application_name = _resolve_new_foundry_application_name(foundry_settings)
     endpoint = _resolve_endpoint(foundry_settings, global_settings)
     responses_api_version = (
@@ -1222,10 +1252,45 @@ def _looks_like_document_context_message(text: str) -> bool:
         "chat-uploaded file",
         "selected document",
         "tabular analysis",
+        "mixed-source evidence handoff",
+        "evidence_envelopes",
+        "[workflow document search context]",
     )
     return any(marker in normalized for marker in markers)
 
 
+def _filter_foundry_document_context_messages(
+    message_history: List[ChatMessageContent],
+    include_document_context: bool = True,
+) -> List[ChatMessageContent]:
+    """Remove document/evidence messages before Foundry transport when opted out."""
+    if include_document_context:
+        return list(message_history or [])
+
+    filtered_messages: List[ChatMessageContent] = []
+    workflow_task_marker = "[workflow task]"
+    for message in list(message_history or []):
+        text = _extract_message_text(message).strip()
+        if not text:
+            continue
+        if not _looks_like_document_context_message(text):
+            filtered_messages.append(message)
+            continue
+
+        marker_index = text.lower().rfind(workflow_task_marker)
+        if marker_index < 0:
+            continue
+        workflow_task = text[marker_index + len(workflow_task_marker):].strip()
+        if not workflow_task:
+            continue
+        filtered_messages.append(ChatMessageContent(
+            role=getattr(message, "role", "user"),
+            content=workflow_task,
+            metadata=getattr(message, "metadata", {}) or {},
+        ))
+    return filtered_messages
+
+
 def _build_foundry_workflow_input_text(
     message_history: List[ChatMessageContent],
     max_context_chars: Optional[int] = None,
@@ -1237,7 +1302,13 @@ def _build_foundry_workflow_input_text(
         if not text:
             continue
         if not include_document_context and _looks_like_document_context_message(text):
-            continue
+            workflow_task_marker = "[Workflow task]"
+            workflow_task_index = text.rfind(workflow_task_marker)
+            if workflow_task_index < 0:
+                continue
+            text = text[workflow_task_index + len(workflow_task_marker):].strip()
+            if not text:
+                continue
         role_value = getattr(message, "role", "user")
         role = str(role_value).strip().lower() or "user"
         if role.startswith("authorrole."):
@@ -1800,6 +1871,8 @@ def _collect_foundry_response_file_inputs(
     foundry_settings: Dict[str, Any],
     metadata: Dict[str, Any],
 ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    if not _coerce_bool(foundry_settings.get("include_document_context"), True):
+        return [], []
     if not _coerce_bool(foundry_settings.get("include_file_inputs"), True):
         return [], []